diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b27e28d2..fc02dbde5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,8 @@ on: pull_request: branches: - "*" +env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true jobs: Get-CI-Image-Tag: @@ -17,7 +19,7 @@ jobs: needs: Get-CI-Image-Tag strategy: matrix: - java: [11, 17] + java: [21] os: [ ubuntu-latest ] name: Build and Test security-analytics with JDK ${{ matrix.java }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -26,11 +28,13 @@ jobs: # this image tag is subject to change as more dependencies and updates will arrive over time image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} # need to switch to root so that github actions can install runner binary on container without permission issues. - options: --user root + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} steps: + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v1 @@ -40,7 +44,7 @@ jobs: - name: Build and Test run: | chown -R 1000:1000 `pwd` - su `id -un 1000` -c "whoami && java -version && ./gradlew build" + su `id -un 1000` -c "whoami && java -version && ./gradlew build --refresh-dependencies" - name: Create Artifact Path run: | @@ -48,22 +52,24 @@ jobs: cp ./build/distributions/*.zip security-analytics-artifacts - name: Upload Coverage Report - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} - name: Upload failed logs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: failure() with: name: logs-ubuntu path: build/testclusters/integTest-*/logs/* + overwrite: true - name: Upload Artifacts - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v4 with: name: security-analytics-plugin-${{ matrix.os }} path: security-analytics-artifacts + overwrite: true build-windows-macos: env: @@ -71,7 +77,7 @@ jobs: WORKING_DIR: ${{ matrix.working_directory }}. strategy: matrix: - java: [11, 17] + java: [21] os: [ windows-latest, macos-latest ] include: - os: windows-latest @@ -86,7 +92,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 # This is a hack, but this step creates a link to the X: mounted drive, which makes the path # short enough to work on Windows @@ -101,7 +107,7 @@ jobs: - name: Build and Test working-directory: ${{ env.WORKING_DIR }} - run: ./gradlew build ${{ env.BUILD_ARGS }} + run: ./gradlew build --refresh-dependencies ${{ env.BUILD_ARGS }} env: _JAVA_OPTIONS: ${{ matrix.os_java_options }} @@ -111,21 +117,24 @@ jobs: cp ./build/distributions/*.zip security-analytics-artifacts - name: Upload failed logs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: ${{ failure() && matrix.os == 'macos-latest' }} with: name: logs-mac path: build/testclusters/integTest-*/logs/* + overwrite: true - name: Upload failed logs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: ${{ failure() && matrix.os == 'windows-latest' }} with: name: logs-windows path: build\testclusters\integTest-*\logs\* + overwrite: true - name: Upload Artifacts - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v4 with: name: security-analytics-plugin-${{ matrix.os }} path: security-analytics-artifacts + overwrite: true diff --git a/.github/workflows/maven-publish.yml b/.github/workflows/maven-publish.yml index 609ef7b82..9f2c40db0 100644 --- a/.github/workflows/maven-publish.yml +++ b/.github/workflows/maven-publish.yml @@ -25,7 +25,7 @@ jobs: with: distribution: temurin # Temurin is a distribution of adoptium java-version: 17 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: aws-actions/configure-aws-credentials@v1 with: role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }} diff --git a/.github/workflows/multi-node-test-workflow.yml b/.github/workflows/multi-node-test-workflow.yml index 2f2134947..b92a20469 100644 --- a/.github/workflows/multi-node-test-workflow.yml +++ b/.github/workflows/multi-node-test-workflow.yml @@ -7,7 +7,6 @@ on: push: branches: - "*" - jobs: Get-CI-Image-Tag: uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main @@ -18,7 +17,7 @@ jobs: needs: Get-CI-Image-Tag strategy: matrix: - java: [ 11, 17, 21 ] + java: [ 21 ] # Job name name: Build and test Security Analytics on linux # This job runs on Linux @@ -28,9 +27,11 @@ jobs: # this image tag is subject to change as more dependencies and updates will arrive over time image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} # need to switch to root so that github actions can install runner binary on container without permission issues. - options: --user root + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} steps: + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} # This step uses the setup-java Github action: https://github.com/actions/setup-java - name: Set Up JDK ${{ matrix.java }} uses: actions/setup-java@v1 @@ -38,13 +39,13 @@ jobs: java-version: ${{ matrix.java }} # This step uses the checkout Github action: https://github.com/actions/checkout - name: Checkout Branch - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Run integration tests with multi node config run: | chown -R 1000:1000 `pwd` su `id -un 1000` -c "./gradlew integTest -PnumNodes=3" - name: Upload failed logs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: failure() with: name: logs diff --git a/.github/workflows/security-test-workflow.yml b/.github/workflows/security-test-workflow.yml index 8c3a38629..47a2b9db0 100644 --- a/.github/workflows/security-test-workflow.yml +++ b/.github/workflows/security-test-workflow.yml @@ -9,12 +9,13 @@ on: - "*" env: OPENSEARCH_INITIAL_ADMIN_PASSWORD: myStrongPassword123! + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true jobs: build: strategy: matrix: - java: [ 11, 17, 21 ] + java: [ 21 ] # Job name name: Build and test SecurityAnalytics # This job runs on Linux @@ -27,7 +28,7 @@ jobs: java-version: ${{ matrix.java }} # This step uses the checkout Github action: https://github.com/actions/checkout - name: Checkout Branch - uses: actions/checkout@v2 + uses: actions/checkout@v4 # This step uses the setup-java Github action: https://github.com/actions/setup-java - name: Set Up JDK ${{ matrix.java }} uses: actions/setup-java@v1 diff --git a/build.gradle b/build.gradle index f29c78554..dd27b9cce 100644 --- a/build.gradle +++ b/build.gradle @@ -7,21 +7,31 @@ import org.opensearch.gradle.test.RestIntegTestTask buildscript { ext { - opensearch_version = System.getProperty("opensearch.version", "2.15.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.19.1-SNAPSHOT") isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") version_tokens = opensearch_version.tokenize('-') opensearch_build = version_tokens[0] + '.0' plugin_no_snapshot = opensearch_build opensearch_no_snapshot = opensearch_version.replace("-SNAPSHOT","") + sa_commons_version = '1.0.0' if (buildVersionQualifier) { opensearch_build += "-${buildVersionQualifier}" } + + alerting_spi_build = opensearch_build + alerting_spi_build += "-SNAPSHOT" if (isSnapshot) { opensearch_build += "-SNAPSHOT" + + // TODO consider enabling snapshot options once SA commons is published to maven central +// sa_commons_version += "-SNAPSHOT" } common_utils_version = System.getProperty("common_utils.version", opensearch_build) - kotlin_version = '1.6.10' + kotlin_version = '1.8.21' + + sa_commons_file_name = "security-analytics-commons-${sa_commons_version}.jar" + sa_commons_file_path = "${project.rootDir}/${sa_commons_file_name}" } repositories { @@ -54,7 +64,7 @@ ext { noticeFile = rootProject.file('NOTICE') } -licenseHeaders.enabled = true +licenseHeaders.enabled = false testingConventions.enabled = false forbiddenApis.ignoreFailures = true @@ -68,7 +78,7 @@ opensearchplugin { name 'opensearch-security-analytics' description 'OpenSearch Security Analytics plugin' classname 'org.opensearch.securityanalytics.SecurityAnalyticsPlugin' - extendedPlugins = ['opensearch-job-scheduler'] + extendedPlugins = ['opensearch-job-scheduler', 'opensearch-alerting'] } javaRestTest { @@ -150,7 +160,7 @@ configurations { resolutionStrategy { // for spotless transitive dependency CVE force "org.eclipse.platform:org.eclipse.core.runtime:3.29.0" - force "com.google.guava:guava:32.1.2-jre" + force "com.google.guava:guava:32.1.3-jre" } } } @@ -158,19 +168,28 @@ configurations { dependencies { javaRestTestImplementation project.sourceSets.main.runtimeClasspath implementation group: 'org.apache.commons', name: 'commons-lang3', version: "${versions.commonslang}" - implementation "org.antlr:antlr4-runtime:4.10.1" - implementation "com.cronutils:cron-utils:9.1.6" - api "org.opensearch:common-utils:${common_utils_version}@jar" - api "org.opensearch.client:opensearch-rest-client:${opensearch_version}" - implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}" + compileOnly "org.antlr:antlr4-runtime:4.10.1" + compileOnly "com.cronutils:cron-utils:9.1.7" + compileOnly "org.opensearch:common-utils:${common_utils_version}@jar" + compileOnly "org.opensearch.client:opensearch-rest-client:${opensearch_version}" + compileOnly "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}" compileOnly "org.opensearch:opensearch-job-scheduler-spi:${opensearch_build}" + compileOnly "org.opensearch.alerting:alerting-spi:${alerting_spi_build}" implementation "org.apache.commons:commons-csv:1.10.0" + compileOnly "com.google.guava:guava:32.1.3-jre" + + // TODO uncomment once SA commons is published to maven central +// api "org.opensearch:security-analytics-commons:${sa_commons_version}@jar" + + // TODO remove once SA commons is published to maven central + api files(sa_commons_file_path) // Needed for integ tests zipArchive group: 'org.opensearch.plugin', name:'alerting', version: "${opensearch_build}" zipArchive group: 'org.opensearch.plugin', name:'opensearch-notifications-core', version: "${opensearch_build}" zipArchive group: 'org.opensearch.plugin', name:'notifications', version: "${opensearch_build}" zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}" + testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1' } // RPM & Debian build @@ -364,6 +383,12 @@ afterEvaluate { into opensearchplugin.name } + // TODO remove once SA commons is published to maven central + from(project.rootDir) { + include sa_commons_file_name + into opensearchplugin.name + } + user 'root' permissionGroup 'root' fileMode 0644 diff --git a/release-notes/opensearch-security-analytics.release-notes-2.15.0.0.md b/release-notes/opensearch-security-analytics.release-notes-2.15.0.0.md index a93be647e..9bd0879dd 100644 --- a/release-notes/opensearch-security-analytics.release-notes-2.15.0.0.md +++ b/release-notes/opensearch-security-analytics.release-notes-2.15.0.0.md @@ -2,6 +2,10 @@ Compatible with OpenSearch 2.15.0 +### Features +* Alerts in correlations [Experminental] ([#1040](https://github.com/opensearch-project/security-analytics/pull/1040)) +* Alerts in Correlations Part 2 ([#1062](https://github.com/opensearch-project/security-analytics/pull/1062)) + ### Maintenance * Increment version to 2.15.0-SNAPSHOT. ([#1055](https://github.com/opensearch-project/security-analytics/pull/1055)) * Fix codecov calculation ([#1021](https://github.com/opensearch-project/security-analytics/pull/1021)) @@ -12,4 +16,4 @@ Compatible with OpenSearch 2.15.0 * Change default filter to time based fields ([#1030](https://github.com/opensearch-project/security-analytics/pull/1030)) ### Documentation -* Added 2.15.0 release notes. ([#1061](https://github.com/opensearch-project/security-analytics/pull/1061)) \ No newline at end of file +* Added 2.15.0 release notes. ([#1061](https://github.com/opensearch-project/security-analytics/pull/1061)) diff --git a/release-notes/opensearch-security-analytics.release-notes-2.16.0.0.md b/release-notes/opensearch-security-analytics.release-notes-2.16.0.0.md new file mode 100644 index 000000000..d67712959 --- /dev/null +++ b/release-notes/opensearch-security-analytics.release-notes-2.16.0.0.md @@ -0,0 +1,35 @@ +## Version 2.16.0.0 2024-07-23 + +Compatible with OpenSearch 2.16.0 + +### Features +* Threat Intel Analytics ([#1098](https://github.com/opensearch-project/security-analytics/pull/1098)) + +### Maintenance +* Incremented version to 2.16.0. ([#1197](https://github.com/opensearch-project/security-analytics/pull/1197)) +* Fix build CI error due to action runner env upgrade node 20 ([#1143](https://github.com/opensearch-project/security-analytics/pull/1143)) + +### Enhancement +* added correlationAlert integ tests ([#1099](https://github.com/opensearch-project/security-analytics/pull/1099)) +* add filter to list ioc api to fetch only from available and refreshing apis. null check for alias of ioc indices ([#1131](https://github.com/opensearch-project/security-analytics/pull/1131)) +* Changes threat intel default store config model ([#1133](https://github.com/opensearch-project/security-analytics/pull/1133)) +* adds new tif source config type - url download ([#1142](https://github.com/opensearch-project/security-analytics/pull/1142)) + +### Bug Fixes +* pass integ tests ([#1082](https://github.com/opensearch-project/security-analytics/pull/1082)) +* set blank response when indexNotFound exception ([#1125](https://github.com/opensearch-project/security-analytics/pull/1125)) +* throw error when no iocs are stored due to incompatible ioc types from S3 downloaded iocs file ([#1129](https://github.com/opensearch-project/security-analytics/pull/1129)) +* fix findingIds filter on ioc findings search api ([#1130](https://github.com/opensearch-project/security-analytics/pull/1130)) +* Adjusted IOCTypes usage ([#1156](https://github.com/opensearch-project/security-analytics/pull/1156)) +* Fix the job scheduler parser, action listeners, and multi-node test ([#1157](https://github.com/opensearch-project/security-analytics/pull/1157)) +* ListIOCs API to return number of findings per IOC ([#1163](https://github.com/opensearch-project/security-analytics/pull/1163)) +* Ioc upload integ tests and fix update ([#1162](https://github.com/opensearch-project/security-analytics/pull/1162)) +* [BUG] Resolve aliases in monitor input to concrete indices before computing ioc-containing fields from concrete index docs ([#1173](https://github.com/opensearch-project/security-analytics/pull/1173)) +* Enum fix ([#1178](https://github.com/opensearch-project/security-analytics/pull/1178)) +* fix bug: threat intel monitor finding doesnt contain all doc_ids containing malicious IOC ([#1184](https://github.com/opensearch-project/security-analytics/pull/1184)) +* Fixed bulk indexing for IOCs ([#1187](https://github.com/opensearch-project/security-analytics/pull/1187)) +* Fix ioc upload update behavior and change error response ([#1192](https://github.com/opensearch-project/security-analytics/pull/1192)) +* Catch and wrap exceptions. ([#1198](https://github.com/opensearch-project/security-analytics/pull/1198)) + +### Documentation +* Added 2.16.0 release notes. ([#1196](https://github.com/opensearch-project/security-analytics/pull/1196)) \ No newline at end of file diff --git a/release-notes/opensearch-security-analytics.release-notes-2.17.0.0.md b/release-notes/opensearch-security-analytics.release-notes-2.17.0.0.md new file mode 100644 index 000000000..287ccda70 --- /dev/null +++ b/release-notes/opensearch-security-analytics.release-notes-2.17.0.0.md @@ -0,0 +1,27 @@ +## Version 2.17.0.0 2024-09-05 + +Compatible with OpenSearch 2.17.0 + +### Maintenance +* update build.gradle to use alerting-spi snapshot version ([#1217](https://github.com/opensearch-project/security-analytics/pull/1217)) + +### Enhancement +* added triggers in getDetectors API response ([#1226](https://github.com/opensearch-project/security-analytics/pull/1226)) +* secure rest tests for threat intel monitor apis ([#1212](https://github.com/opensearch-project/security-analytics/pull/1212)) + +### Bug Fixes +* Adds user validation for threat intel transport layer classes and stashes the thread context for all system index interactions ([#1207](https://github.com/opensearch-project/security-analytics/pull/1207)) +* fix mappings integ tests ([#1213](https://github.com/opensearch-project/security-analytics/pull/1213)) +* Bug fixes for threat intel ([#1223](https://github.com/opensearch-project/security-analytics/pull/1223)) +* make threat intel run with standard detectors ([#1234](https://github.com/opensearch-project/security-analytics/pull/1234)) +* Fixed searchString bug. Removed nested IOC mapping structure. ([#1239](https://github.com/opensearch-project/security-analytics/pull/1239)) +* adds toggling refresh disable/enable for deactivate/activate operation while updating URL_DOWNLOAD type configs ([#1240](https://github.com/opensearch-project/security-analytics/pull/1240)) +* Make threat intel source config release lock event driven ([#1254](https://github.com/opensearch-project/security-analytics/pull/1254)) +* Fix S3 validation errors not caught by action listener ([#1257](https://github.com/opensearch-project/security-analytics/pull/1257)) +* Clean up empty IOC indices created by failed source configs ([#1267](https://github.com/opensearch-project/security-analytics/pull/1267)) +* Fix threat intel multinode tests ([#1274](https://github.com/opensearch-project/security-analytics/pull/1274)) +* Update threat intel job mapping to new version ([#1272](https://github.com/opensearch-project/security-analytics/pull/1272)) +* Stash context for List IOCs Api ([#1278](https://github.com/opensearch-project/security-analytics/pull/1278)) + +### Documentation +* Added 2.17.0 release notes. ([#1290](https://github.com/opensearch-project/security-analytics/pull/1290)) \ No newline at end of file diff --git a/release-notes/opensearch-security-analytics.release-notes-2.17.1.0.md b/release-notes/opensearch-security-analytics.release-notes-2.17.1.0.md new file mode 100644 index 000000000..820c79255 --- /dev/null +++ b/release-notes/opensearch-security-analytics.release-notes-2.17.1.0.md @@ -0,0 +1,15 @@ +## Version 2.17.1.0 2024-09-27 + +Compatible with OpenSearch 2.17.1 + +### Maintenance +* upgrade upload artifacts ([#1305](https://github.com/opensearch-project/security-analytics/pull/1305)) +* Incremented version to 2.17.1 ([#1304](https://github.com/opensearch-project/security-analytics/pull/1304)) + +### Bug Fixes +* [Alerts in Correlations] Stash context for system index ([#1297](https://github.com/opensearch-project/security-analytics/pull/1297)) +* threat intel monitor bug fixes ([#1317](https://github.com/opensearch-project/security-analytics/pull/1317)) + + +### Documentation +* Added 2.17.1 release notes. ([#1331](https://github.com/opensearch-project/security-analytics/pull/1331)) diff --git a/release-notes/opensearch-security-analytics.release-notes-2.18.0.0.md b/release-notes/opensearch-security-analytics.release-notes-2.18.0.0.md new file mode 100644 index 000000000..a389a2128 --- /dev/null +++ b/release-notes/opensearch-security-analytics.release-notes-2.18.0.0.md @@ -0,0 +1,24 @@ +## Version 2.18.0.0 2024-10-28 + +Compatible with OpenSearch 2.18.0 + +### Maintenance +* Incremented version to 2.18.0 ([#1314](https://github.com/opensearch-project/security-analytics/pull/1314)) +* update to lucene 9.12 ([#1349](https://github.com/opensearch-project/security-analytics/pull/1349)) + +### Refactoring +* separate doc-level monitor query indices created by detectors ([#1324](https://github.com/opensearch-project/security-analytics/pull/1324)) +* update number of replicas of system indices to 1-20 and number of primary shards for system indices to 1 ([#1358](https://github.com/opensearch-project/security-analytics/pull/1358)) +* update min number of replicas to 0 ([#1364](https://github.com/opensearch-project/security-analytics/pull/1364)) +* updated dedicated query index settings to true ([#1365](https://github.com/opensearch-project/security-analytics/pull/1365)) +* set the refresh policy to IMMEDIATE when updating correlation alerts ([#1382](https://github.com/opensearch-project/security-analytics/pull/1382)) + +### Bug Fixes +* remove redundant logic to fix OS launch exception and updates actions/upload-artifac2 to @V3 ([#1303](https://github.com/opensearch-project/security-analytics/pull/1303)) +* Add null check while adding fetched iocs into per-indicator-type map ([#1335](https://github.com/opensearch-project/security-analytics/pull/1335)) +* Fix notifications listener leak in threat intel monitor ([#1361](https://github.com/opensearch-project/security-analytics/pull/1361)) +* [Bug] Fixed ListIOCs number of findings cap. ([#1373](https://github.com/opensearch-project/security-analytics/pull/1373)) +* [Bug] Add exists check for IOCs index. ([#1392](https://github.com/opensearch-project/security-analytics/pull/1392)) + +### Documentation +* Added 2.18.0 release notes. ([#1399](https://github.com/opensearch-project/security-analytics/pull/1399)) \ No newline at end of file diff --git a/release-notes/opensearch-security-analytics.release-notes-2.19.0.0.md b/release-notes/opensearch-security-analytics.release-notes-2.19.0.0.md new file mode 100644 index 000000000..477322e84 --- /dev/null +++ b/release-notes/opensearch-security-analytics.release-notes-2.19.0.0.md @@ -0,0 +1,21 @@ +## Version 2.19.0.0 2025-02-03 +Compatible with OpenSearch 2.19.0 + +### Maintenance +* Incremented version to 2.19.0 ([#1444](https://github.com/opensearch-project/security-analytics/pull/1444)) +* Fix CVE-2024-47535. ([#1460](https://github.com/opensearch-project/security-analytics/pull/1460)) + +### Refactoring +* optimize sigma aggregation rule based detectors execution workflow ([#1418](https://github.com/opensearch-project/security-analytics/pull/1418)) +* Adding various OCSF 1.1 fields to log type static mappings ([#1403](https://github.com/opensearch-project/security-analytics/pull/1403)) + +### Bug Fixes +* Add validation for threat intel source config ([#1393](https://github.com/opensearch-project/security-analytics/pull/1393)) +* fix detector to work for trigger conditions filtering on aggregation rules ([#1423](https://github.com/opensearch-project/security-analytics/pull/1423)) +* fixes the duplicate alerts generated by Aggregation Sigma Roles ([#1424](https://github.com/opensearch-project/security-analytics/pull/1424)) +* OCSF1.1 Fixes ([#1439](https://github.com/opensearch-project/security-analytics/pull/1439)) +* Added catch for unexpected inputs. ([#1442](https://github.com/opensearch-project/security-analytics/pull/1442)) +* Refactored flaky test. ([#1464](https://github.com/opensearch-project/security-analytics/pull/1464)) + +### Documentation +* Added 2.19.0 release notes. ([#1468](https://github.com/opensearch-project/security-analytics/pull/1468)) \ No newline at end of file diff --git a/security-analytics-commons-1.0.0.jar b/security-analytics-commons-1.0.0.jar new file mode 100644 index 000000000..094a79102 Binary files /dev/null and b/security-analytics-commons-1.0.0.jar differ diff --git a/src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionBaseListener.java b/src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionBaseListener.java index f30585e27..54bdc8329 100644 --- a/src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionBaseListener.java +++ b/src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionBaseListener.java @@ -1,8 +1,4 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -// Generated from Condition.g4 by ANTLR 4.10.1 +// Generated from java-escape by ANTLR 4.11.1 package org.opensearch.securityanalytics.rules.condition; import org.antlr.v4.runtime.ParserRuleContext; @@ -14,6 +10,7 @@ * which can be extended to create a listener which only needs to handle a subset * of the available methods. */ +@SuppressWarnings("CheckReturnValue") public class ConditionBaseListener implements ConditionListener { /** * {@inheritDoc} diff --git a/src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionBaseVisitor.java b/src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionBaseVisitor.java index b7294de26..6f6962ecc 100644 --- a/src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionBaseVisitor.java +++ b/src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionBaseVisitor.java @@ -1,8 +1,4 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -// Generated from Condition.g4 by ANTLR 4.10.1 +// Generated from java-escape by ANTLR 4.11.1 package org.opensearch.securityanalytics.rules.condition; import org.antlr.v4.runtime.tree.AbstractParseTreeVisitor; @@ -14,6 +10,7 @@ * @param The return type of the visit operation. Use {@link Void} for * operations with no return type. */ +@SuppressWarnings("CheckReturnValue") public class ConditionBaseVisitor extends AbstractParseTreeVisitor implements ConditionVisitor { /** * {@inheritDoc} diff --git a/src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionLexer.java b/src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionLexer.java index b51f795f5..d7eae93a1 100644 --- a/src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionLexer.java +++ b/src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionLexer.java @@ -1,8 +1,4 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -// Generated from Condition.g4 by ANTLR 4.10.1 +// Generated from java-escape by ANTLR 4.11.1 package org.opensearch.securityanalytics.rules.condition; import org.antlr.v4.runtime.Lexer; import org.antlr.v4.runtime.CharStream; @@ -13,9 +9,9 @@ import org.antlr.v4.runtime.dfa.DFA; import org.antlr.v4.runtime.misc.*; -@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) +@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue"}) public class ConditionLexer extends Lexer { - static { RuntimeMetaData.checkVersion("4.10.1", RuntimeMetaData.VERSION); } + static { RuntimeMetaData.checkVersion("4.11.1", RuntimeMetaData.VERSION); } protected static final DFA[] _decisionToDFA; protected static final PredictionContextCache _sharedContextCache = diff --git a/src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionListener.java b/src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionListener.java index 1f595edd6..fd8403273 100644 --- a/src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionListener.java +++ b/src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionListener.java @@ -1,8 +1,4 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -// Generated from Condition.g4 by ANTLR 4.10.1 +// Generated from java-escape by ANTLR 4.11.1 package org.opensearch.securityanalytics.rules.condition; import org.antlr.v4.runtime.tree.ParseTreeListener; diff --git a/src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionParser.java b/src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionParser.java index 865c6bf21..01b869f4b 100644 --- a/src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionParser.java +++ b/src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionParser.java @@ -1,8 +1,4 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -// Generated from Condition.g4 by ANTLR 4.10.1 +// Generated from java-escape by ANTLR 4.11.1 package org.opensearch.securityanalytics.rules.condition; import org.antlr.v4.runtime.atn.*; import org.antlr.v4.runtime.dfa.DFA; @@ -13,9 +9,9 @@ import java.util.Iterator; import java.util.ArrayList; -@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) +@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue"}) public class ConditionParser extends Parser { - static { RuntimeMetaData.checkVersion("4.10.1", RuntimeMetaData.VERSION); } + static { RuntimeMetaData.checkVersion("4.11.1", RuntimeMetaData.VERSION); } protected static final DFA[] _decisionToDFA; protected static final PredictionContextCache _sharedContextCache = @@ -78,7 +74,7 @@ public Vocabulary getVocabulary() { } @Override - public String getGrammarFileName() { return "Condition.g4"; } + public String getGrammarFileName() { return "java-escape"; } @Override public String[] getRuleNames() { return ruleNames; } @@ -94,6 +90,7 @@ public ConditionParser(TokenStream input) { _interp = new ParserATNSimulator(this,_ATN,_decisionToDFA,_sharedContextCache); } + @SuppressWarnings("CheckReturnValue") public static class StartContext extends ParserRuleContext { public ExpressionContext expression() { return getRuleContext(ExpressionContext.class,0); @@ -138,6 +135,7 @@ public final StartContext start() throws RecognitionException { return _localctx; } + @SuppressWarnings("CheckReturnValue") public static class ExpressionContext extends ParserRuleContext { public ExpressionContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); @@ -149,6 +147,7 @@ public void copyFrom(ExpressionContext ctx) { super.copyFrom(ctx); } } + @SuppressWarnings("CheckReturnValue") public static class OrExpressionContext extends ExpressionContext { public ExpressionContext left; public Token operator; @@ -175,6 +174,7 @@ public T accept(ParseTreeVisitor visitor) { else return visitor.visitChildren(this); } } + @SuppressWarnings("CheckReturnValue") public static class IdentOrSelectExpressionContext extends ExpressionContext { public TerminalNode SELECTOR() { return getToken(ConditionParser.SELECTOR, 0); } public TerminalNode IDENTIFIER() { return getToken(ConditionParser.IDENTIFIER, 0); } @@ -193,6 +193,7 @@ public T accept(ParseTreeVisitor visitor) { else return visitor.visitChildren(this); } } + @SuppressWarnings("CheckReturnValue") public static class AndExpressionContext extends ExpressionContext { public ExpressionContext left; public Token operator; @@ -219,6 +220,7 @@ public T accept(ParseTreeVisitor visitor) { else return visitor.visitChildren(this); } } + @SuppressWarnings("CheckReturnValue") public static class NotExpressionContext extends ExpressionContext { public TerminalNode NOT() { return getToken(ConditionParser.NOT, 0); } public ExpressionContext expression() { @@ -239,6 +241,7 @@ public T accept(ParseTreeVisitor visitor) { else return visitor.visitChildren(this); } } + @SuppressWarnings("CheckReturnValue") public static class ParenExpressionContext extends ExpressionContext { public ExpressionContext inner; public TerminalNode LPAREN() { return getToken(ConditionParser.LPAREN, 0); } diff --git a/src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionVisitor.java b/src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionVisitor.java index 5813f169d..1bf3cc169 100644 --- a/src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionVisitor.java +++ b/src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionVisitor.java @@ -1,8 +1,4 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -// Generated from Condition.g4 by ANTLR 4.10.1 +// Generated from java-escape by ANTLR 4.11.1 package org.opensearch.securityanalytics.rules.condition; import org.antlr.v4.runtime.tree.ParseTreeVisitor; diff --git a/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationBaseListener.java b/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationBaseListener.java index 40111dc84..eef40b417 100644 --- a/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationBaseListener.java +++ b/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationBaseListener.java @@ -1,8 +1,4 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -// Generated from Aggregation.g4 by ANTLR 4.10.1 +// Generated from java-escape by ANTLR 4.11.1 package org.opensearch.securityanalytics.rules.condition.aggregation; import org.antlr.v4.runtime.ParserRuleContext; @@ -14,6 +10,7 @@ * which can be extended to create a listener which only needs to handle a subset * of the available methods. */ +@SuppressWarnings("CheckReturnValue") public class AggregationBaseListener implements AggregationListener { /** * {@inheritDoc} diff --git a/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationBaseVisitor.java b/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationBaseVisitor.java index 9bb6289a7..736ee6af3 100644 --- a/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationBaseVisitor.java +++ b/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationBaseVisitor.java @@ -1,8 +1,4 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -// Generated from Aggregation.g4 by ANTLR 4.10.1 +// Generated from java-escape by ANTLR 4.11.1 package org.opensearch.securityanalytics.rules.condition.aggregation; import org.antlr.v4.runtime.tree.AbstractParseTreeVisitor; @@ -14,6 +10,7 @@ * @param The return type of the visit operation. Use {@link Void} for * operations with no return type. */ +@SuppressWarnings("CheckReturnValue") public class AggregationBaseVisitor extends AbstractParseTreeVisitor implements AggregationVisitor { /** * {@inheritDoc} diff --git a/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationLexer.java b/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationLexer.java index 115766bb1..887bf9735 100644 --- a/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationLexer.java +++ b/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationLexer.java @@ -1,8 +1,4 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -// Generated from Aggregation.g4 by ANTLR 4.10.1 +// Generated from java-escape by ANTLR 4.11.1 package org.opensearch.securityanalytics.rules.condition.aggregation; import org.antlr.v4.runtime.Lexer; import org.antlr.v4.runtime.CharStream; @@ -13,9 +9,9 @@ import org.antlr.v4.runtime.dfa.DFA; import org.antlr.v4.runtime.misc.*; -@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) +@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue"}) public class AggregationLexer extends Lexer { - static { RuntimeMetaData.checkVersion("4.10.1", RuntimeMetaData.VERSION); } + static { RuntimeMetaData.checkVersion("4.11.1", RuntimeMetaData.VERSION); } protected static final DFA[] _decisionToDFA; protected static final PredictionContextCache _sharedContextCache = diff --git a/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationListener.java b/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationListener.java index e1dda0939..b2ee39d55 100644 --- a/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationListener.java +++ b/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationListener.java @@ -1,8 +1,4 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -// Generated from Aggregation.g4 by ANTLR 4.10.1 +// Generated from java-escape by ANTLR 4.11.1 package org.opensearch.securityanalytics.rules.condition.aggregation; import org.antlr.v4.runtime.tree.ParseTreeListener; diff --git a/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationParser.java b/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationParser.java index 71c98cc09..03493139b 100644 --- a/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationParser.java +++ b/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationParser.java @@ -1,8 +1,4 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -// Generated from Aggregation.g4 by ANTLR 4.10.1 +// Generated from java-escape by ANTLR 4.11.1 package org.opensearch.securityanalytics.rules.condition.aggregation; import org.antlr.v4.runtime.atn.*; import org.antlr.v4.runtime.dfa.DFA; @@ -13,9 +9,9 @@ import java.util.Iterator; import java.util.ArrayList; -@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) +@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue"}) public class AggregationParser extends Parser { - static { RuntimeMetaData.checkVersion("4.10.1", RuntimeMetaData.VERSION); } + static { RuntimeMetaData.checkVersion("4.11.1", RuntimeMetaData.VERSION); } protected static final DFA[] _decisionToDFA; protected static final PredictionContextCache _sharedContextCache = @@ -82,7 +78,7 @@ public Vocabulary getVocabulary() { } @Override - public String getGrammarFileName() { return "Aggregation.g4"; } + public String getGrammarFileName() { return "java-escape"; } @Override public String[] getRuleNames() { return ruleNames; } @@ -98,6 +94,7 @@ public AggregationParser(TokenStream input) { _interp = new ParserATNSimulator(this,_ATN,_decisionToDFA,_sharedContextCache); } + @SuppressWarnings("CheckReturnValue") public static class Comparison_exprContext extends ParserRuleContext { public Comparison_exprContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); @@ -109,6 +106,7 @@ public void copyFrom(Comparison_exprContext ctx) { super.copyFrom(ctx); } } + @SuppressWarnings("CheckReturnValue") public static class ComparisonExpressionWithOperatorContext extends Comparison_exprContext { public List comparison_operand() { return getRuleContexts(Comparison_operandContext.class); @@ -161,6 +159,7 @@ public final Comparison_exprContext comparison_expr() throws RecognitionExceptio return _localctx; } + @SuppressWarnings("CheckReturnValue") public static class Comparison_operandContext extends ParserRuleContext { public Agg_exprContext agg_expr() { return getRuleContext(Agg_exprContext.class,0); @@ -205,6 +204,7 @@ public final Comparison_operandContext comparison_operand() throws RecognitionEx return _localctx; } + @SuppressWarnings("CheckReturnValue") public static class Comp_operatorContext extends ParserRuleContext { public TerminalNode GT() { return getToken(AggregationParser.GT, 0); } public TerminalNode GE() { return getToken(AggregationParser.GE, 0); } @@ -239,7 +239,7 @@ public final Comp_operatorContext comp_operator() throws RecognitionException { { setState(20); _la = _input.LA(1); - if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << GT) | (1L << GE) | (1L << LT) | (1L << LE) | (1L << EQ))) != 0)) ) { + if ( !(((_la) & ~0x3f) == 0 && ((1L << _la) & 62L) != 0) ) { _errHandler.recoverInline(this); } else { @@ -260,6 +260,7 @@ public final Comp_operatorContext comp_operator() throws RecognitionException { return _localctx; } + @SuppressWarnings("CheckReturnValue") public static class Agg_operatorContext extends ParserRuleContext { public TerminalNode COUNT() { return getToken(AggregationParser.COUNT, 0); } public TerminalNode SUM() { return getToken(AggregationParser.SUM, 0); } @@ -294,7 +295,7 @@ public final Agg_operatorContext agg_operator() throws RecognitionException { { setState(22); _la = _input.LA(1); - if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << COUNT) | (1L << SUM) | (1L << MIN) | (1L << MAX) | (1L << AVG))) != 0)) ) { + if ( !(((_la) & ~0x3f) == 0 && ((1L << _la) & 1984L) != 0) ) { _errHandler.recoverInline(this); } else { @@ -315,6 +316,7 @@ public final Agg_operatorContext agg_operator() throws RecognitionException { return _localctx; } + @SuppressWarnings("CheckReturnValue") public static class Groupby_exprContext extends ParserRuleContext { public TerminalNode IDENTIFIER() { return getToken(AggregationParser.IDENTIFIER, 0); } public Groupby_exprContext(ParserRuleContext parent, int invokingState) { @@ -357,6 +359,7 @@ public final Groupby_exprContext groupby_expr() throws RecognitionException { return _localctx; } + @SuppressWarnings("CheckReturnValue") public static class Agg_exprContext extends ParserRuleContext { public Agg_exprContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); @@ -368,6 +371,7 @@ public void copyFrom(Agg_exprContext ctx) { super.copyFrom(ctx); } } + @SuppressWarnings("CheckReturnValue") public static class AggExpressionNumericEntityContext extends Agg_exprContext { public Numeric_entityContext numeric_entity() { return getRuleContext(Numeric_entityContext.class,0); @@ -387,6 +391,7 @@ public T accept(ParseTreeVisitor visitor) { else return visitor.visitChildren(this); } } + @SuppressWarnings("CheckReturnValue") public static class AggExpressionParensContext extends Agg_exprContext { public Agg_operatorContext agg_operator() { return getRuleContext(Agg_operatorContext.class,0); @@ -486,6 +491,7 @@ public final Agg_exprContext agg_expr() throws RecognitionException { return _localctx; } + @SuppressWarnings("CheckReturnValue") public static class Numeric_entityContext extends ParserRuleContext { public Numeric_entityContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); @@ -497,6 +503,7 @@ public void copyFrom(Numeric_entityContext ctx) { super.copyFrom(ctx); } } + @SuppressWarnings("CheckReturnValue") public static class NumericConstContext extends Numeric_entityContext { public TerminalNode DECIMAL() { return getToken(AggregationParser.DECIMAL, 0); } public NumericConstContext(Numeric_entityContext ctx) { copyFrom(ctx); } @@ -514,6 +521,7 @@ public T accept(ParseTreeVisitor visitor) { else return visitor.visitChildren(this); } } + @SuppressWarnings("CheckReturnValue") public static class NumericVariableContext extends Numeric_entityContext { public TerminalNode IDENTIFIER() { return getToken(AggregationParser.IDENTIFIER, 0); } public NumericVariableContext(Numeric_entityContext ctx) { copyFrom(ctx); } diff --git a/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationVisitor.java b/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationVisitor.java index f441b2bcc..7fd880dee 100644 --- a/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationVisitor.java +++ b/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationVisitor.java @@ -1,8 +1,4 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -// Generated from Aggregation.g4 by ANTLR 4.10.1 +// Generated from java-escape by ANTLR 4.11.1 package org.opensearch.securityanalytics.rules.condition.aggregation; import org.antlr.v4.runtime.tree.ParseTreeVisitor; diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index 2f5f97c50..2762bd5f8 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -4,20 +4,14 @@ */ package org.opensearch.securityanalytics; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.function.Supplier; -import java.util.Optional; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.core.action.ActionListener; import org.opensearch.action.ActionRequest; -import org.opensearch.core.action.ActionResponse; +import org.opensearch.alerting.spi.RemoteMonitorRunner; +import org.opensearch.alerting.spi.RemoteMonitorRunnerExtension; import org.opensearch.client.Client; import org.opensearch.client.node.NodeClient; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; @@ -29,8 +23,12 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsFilter; import org.opensearch.commons.alerting.action.AlertingActions; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.action.ActionResponse; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.index.IndexSettings; @@ -52,15 +50,15 @@ import org.opensearch.rest.RestController; import org.opensearch.rest.RestHandler; import org.opensearch.script.ScriptService; -import org.opensearch.securityanalytics.action.GetAlertsAction; -import org.opensearch.securityanalytics.action.DeleteCorrelationRuleAction; import org.opensearch.securityanalytics.action.AckAlertsAction; -import org.opensearch.securityanalytics.action.CreateIndexMappingsAction; -import org.opensearch.securityanalytics.action.CorrelatedFindingAction; import org.opensearch.securityanalytics.action.AckCorrelationAlertsAction; +import org.opensearch.securityanalytics.action.CorrelatedFindingAction; +import org.opensearch.securityanalytics.action.CreateIndexMappingsAction; +import org.opensearch.securityanalytics.action.DeleteCorrelationRuleAction; import org.opensearch.securityanalytics.action.DeleteCustomLogTypeAction; import org.opensearch.securityanalytics.action.DeleteDetectorAction; import org.opensearch.securityanalytics.action.DeleteRuleAction; +import org.opensearch.securityanalytics.action.GetAlertsAction; import org.opensearch.securityanalytics.action.GetAllRuleCategoriesAction; import org.opensearch.securityanalytics.action.GetCorrelationAlertsAction; import org.opensearch.securityanalytics.action.GetDetectorAction; @@ -72,40 +70,142 @@ import org.opensearch.securityanalytics.action.IndexDetectorAction; import org.opensearch.securityanalytics.action.IndexRuleAction; import org.opensearch.securityanalytics.action.ListCorrelationsAction; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsAction; import org.opensearch.securityanalytics.action.SearchCorrelationRuleAction; import org.opensearch.securityanalytics.action.SearchCustomLogTypeAction; import org.opensearch.securityanalytics.action.SearchDetectorAction; import org.opensearch.securityanalytics.action.SearchRuleAction; +import org.opensearch.securityanalytics.action.TestS3ConnectionAction; import org.opensearch.securityanalytics.action.UpdateIndexMappingsAction; import org.opensearch.securityanalytics.action.ValidateRulesAction; -import org.opensearch.securityanalytics.correlation.index.codec.CorrelationCodecService; import org.opensearch.securityanalytics.correlation.alert.CorrelationAlertService; import org.opensearch.securityanalytics.correlation.alert.notifications.NotificationService; +import org.opensearch.securityanalytics.correlation.index.codec.CorrelationCodecService; import org.opensearch.securityanalytics.correlation.index.mapper.CorrelationVectorFieldMapper; import org.opensearch.securityanalytics.correlation.index.query.CorrelationQueryBuilder; import org.opensearch.securityanalytics.indexmanagment.DetectorIndexManagementService; +import org.opensearch.securityanalytics.jobscheduler.SecurityAnalyticsRunner; import org.opensearch.securityanalytics.logtype.BuiltinLogTypeLoader; import org.opensearch.securityanalytics.logtype.LogTypeService; import org.opensearch.securityanalytics.mapper.IndexTemplateManager; import org.opensearch.securityanalytics.mapper.MapperService; import org.opensearch.securityanalytics.model.CustomLogType; +import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.securityanalytics.model.DetectorInput; +import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; -import org.opensearch.securityanalytics.resthandler.*; -import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService; -import org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataService; +import org.opensearch.securityanalytics.resthandler.RestAcknowledgeAlertsAction; +import org.opensearch.securityanalytics.resthandler.RestAcknowledgeCorrelationAlertsAction; +import org.opensearch.securityanalytics.resthandler.RestCreateIndexMappingsAction; +import org.opensearch.securityanalytics.resthandler.RestDeleteCorrelationRuleAction; +import org.opensearch.securityanalytics.resthandler.RestDeleteCustomLogTypeAction; +import org.opensearch.securityanalytics.resthandler.RestDeleteDetectorAction; +import org.opensearch.securityanalytics.resthandler.RestDeleteRuleAction; +import org.opensearch.securityanalytics.resthandler.RestGetAlertsAction; +import org.opensearch.securityanalytics.resthandler.RestGetAllRuleCategoriesAction; +import org.opensearch.securityanalytics.resthandler.RestGetCorrelationsAlertsAction; +import org.opensearch.securityanalytics.resthandler.RestGetDetectorAction; +import org.opensearch.securityanalytics.resthandler.RestGetFindingsAction; +import org.opensearch.securityanalytics.resthandler.RestGetIndexMappingsAction; +import org.opensearch.securityanalytics.resthandler.RestGetMappingsViewAction; +import org.opensearch.securityanalytics.resthandler.RestIndexCorrelationRuleAction; +import org.opensearch.securityanalytics.resthandler.RestIndexCustomLogTypeAction; +import org.opensearch.securityanalytics.resthandler.RestIndexDetectorAction; +import org.opensearch.securityanalytics.resthandler.RestIndexRuleAction; +import org.opensearch.securityanalytics.resthandler.RestListCorrelationAction; +import org.opensearch.securityanalytics.threatIntel.resthandler.RestListIOCsAction; +import org.opensearch.securityanalytics.resthandler.RestSearchCorrelationAction; +import org.opensearch.securityanalytics.resthandler.RestSearchCorrelationRuleAction; +import org.opensearch.securityanalytics.resthandler.RestSearchCustomLogTypeAction; +import org.opensearch.securityanalytics.resthandler.RestSearchDetectorAction; +import org.opensearch.securityanalytics.resthandler.RestSearchRuleAction; +import org.opensearch.securityanalytics.resthandler.RestTestS3ConnectionAction; +import org.opensearch.securityanalytics.resthandler.RestUpdateIndexMappingsAction; +import org.opensearch.securityanalytics.resthandler.RestValidateRulesAction; +import org.opensearch.securityanalytics.services.STIX2IOCFetchService; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsAction; import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction; -import org.opensearch.securityanalytics.threatIntel.action.TransportPutTIFJobAction; +import org.opensearch.securityanalytics.threatIntel.action.SADeleteTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.SARefreshTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.SASearchTIFSourceConfigsAction; +import org.opensearch.securityanalytics.threatIntel.action.monitor.DeleteThreatIntelMonitorAction; +import org.opensearch.securityanalytics.threatIntel.action.monitor.GetThreatIntelAlertsAction; +import org.opensearch.securityanalytics.threatIntel.action.monitor.IndexThreatIntelMonitorAction; +import org.opensearch.securityanalytics.threatIntel.action.monitor.SearchThreatIntelMonitorAction; +import org.opensearch.securityanalytics.threatIntel.action.monitor.UpdateThreatIntelAlertStatusAction; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.ThreatIntelAlertService; +import org.opensearch.securityanalytics.threatIntel.iocscan.service.SaIoCScanService; +import org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelMonitorRunner; import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobRunner; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobUpdateService; -import org.opensearch.securityanalytics.transport.*; -import org.opensearch.securityanalytics.model.Rule; -import org.opensearch.securityanalytics.model.Detector; -import org.opensearch.securityanalytics.model.DetectorInput; -import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFSourceConfigRunner; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.model.monitor.TransportThreatIntelMonitorFanOutAction; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.resthandler.RestDeleteTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.resthandler.RestGetIocFindingsAction; +import org.opensearch.securityanalytics.threatIntel.resthandler.RestGetTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.resthandler.RestIndexTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.resthandler.RestRefreshTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.resthandler.RestSearchTIFSourceConfigsAction; +import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestDeleteThreatIntelMonitorAction; +import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestGetThreatIntelAlertsAction; +import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestIndexThreatIntelMonitorAction; +import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestSearchThreatIntelMonitorAction; +import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestUpdateThreatIntelAlertsStatusAction; +import org.opensearch.securityanalytics.threatIntel.service.DefaultTifSourceConfigLoaderService; +import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; +import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService; +import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService; +import org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService; +import org.opensearch.securityanalytics.threatIntel.transport.TransportDeleteTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.transport.TransportGetIocFindingsAction; +import org.opensearch.securityanalytics.threatIntel.transport.TransportGetTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.transport.TransportIndexTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.transport.TransportPutTIFJobAction; +import org.opensearch.securityanalytics.threatIntel.transport.TransportRefreshTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.transport.TransportSearchTIFSourceConfigsAction; +import org.opensearch.securityanalytics.threatIntel.transport.monitor.TransportDeleteThreatIntelMonitorAction; +import org.opensearch.securityanalytics.threatIntel.transport.monitor.TransportGetThreatIntelAlertsAction; +import org.opensearch.securityanalytics.threatIntel.transport.monitor.TransportIndexThreatIntelMonitorAction; +import org.opensearch.securityanalytics.threatIntel.transport.monitor.TransportSearchThreatIntelMonitorAction; +import org.opensearch.securityanalytics.threatIntel.transport.monitor.TransportUpdateThreatIntelAlertStatusAction; +import org.opensearch.securityanalytics.transport.TransportAckCorrelationAlertsAction; +import org.opensearch.securityanalytics.transport.TransportAcknowledgeAlertsAction; +import org.opensearch.securityanalytics.transport.TransportCorrelateFindingAction; +import org.opensearch.securityanalytics.transport.TransportCreateIndexMappingsAction; +import org.opensearch.securityanalytics.transport.TransportDeleteCorrelationRuleAction; +import org.opensearch.securityanalytics.transport.TransportDeleteCustomLogTypeAction; +import org.opensearch.securityanalytics.transport.TransportDeleteDetectorAction; +import org.opensearch.securityanalytics.transport.TransportDeleteRuleAction; +import org.opensearch.securityanalytics.transport.TransportGetAlertsAction; +import org.opensearch.securityanalytics.transport.TransportGetAllRuleCategoriesAction; +import org.opensearch.securityanalytics.transport.TransportGetCorrelationAlertsAction; +import org.opensearch.securityanalytics.transport.TransportGetDetectorAction; +import org.opensearch.securityanalytics.transport.TransportGetFindingsAction; +import org.opensearch.securityanalytics.transport.TransportGetIndexMappingsAction; +import org.opensearch.securityanalytics.transport.TransportGetMappingsViewAction; +import org.opensearch.securityanalytics.transport.TransportIndexCorrelationRuleAction; +import org.opensearch.securityanalytics.transport.TransportIndexCustomLogTypeAction; +import org.opensearch.securityanalytics.transport.TransportIndexDetectorAction; +import org.opensearch.securityanalytics.transport.TransportIndexRuleAction; +import org.opensearch.securityanalytics.transport.TransportListCorrelationAction; +import org.opensearch.securityanalytics.threatIntel.transport.TransportListIOCsAction; +import org.opensearch.securityanalytics.transport.TransportSearchCorrelationAction; +import org.opensearch.securityanalytics.transport.TransportSearchCorrelationRuleAction; +import org.opensearch.securityanalytics.transport.TransportSearchCustomLogTypeAction; +import org.opensearch.securityanalytics.transport.TransportSearchDetectorAction; +import org.opensearch.securityanalytics.transport.TransportSearchRuleAction; +import org.opensearch.securityanalytics.transport.TransportTestS3ConnectionAction; +import org.opensearch.securityanalytics.transport.TransportUpdateIndexMappingsAction; +import org.opensearch.securityanalytics.transport.TransportValidateRulesAction; import org.opensearch.securityanalytics.util.CorrelationIndices; import org.opensearch.securityanalytics.util.CorrelationRuleIndices; import org.opensearch.securityanalytics.util.CustomLogTypeIndices; @@ -114,10 +214,21 @@ import org.opensearch.securityanalytics.util.RuleTopicIndices; import org.opensearch.threadpool.ThreadPool; import org.opensearch.watcher.ResourceWatcherService; +import reactor.util.annotation.NonNull; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; -import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; +import static org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE; +import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.SOURCE_CONFIG_FIELD; +import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; +import static org.opensearch.securityanalytics.util.CorrelationIndices.CORRELATION_ALERT_INDEX; -public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin, SystemIndexPlugin, JobSchedulerExtension { +public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin, SystemIndexPlugin, JobSchedulerExtension, RemoteMonitorRunnerExtension { private static final Logger log = LogManager.getLogger(SecurityAnalyticsPlugin.class); @@ -131,11 +242,21 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map public static final String FINDINGS_CORRELATE_URI = FINDINGS_BASE_URI + "/correlate"; public static final String LIST_CORRELATIONS_URI = PLUGINS_BASE_URI + "/correlations"; public static final String CORRELATION_RULES_BASE_URI = PLUGINS_BASE_URI + "/correlation/rules"; + public static final String THREAT_INTEL_BASE_URI = PLUGINS_BASE_URI + "/threat_intel"; + public static final String THREAT_INTEL_SOURCE_URI = PLUGINS_BASE_URI + "/threat_intel/sources"; + public static final String THREAT_INTEL_MONITOR_URI = PLUGINS_BASE_URI + "/threat_intel/monitors"; + public static final String LIST_IOCS_URI = PLUGINS_BASE_URI + "/threat_intel/iocs"; + public static final String THREAT_INTEL_ALERTS_URI = PLUGINS_BASE_URI + "/threat_intel/alerts"; + public static final String THREAT_INTEL_ALERTS_STATUS_URI = PLUGINS_BASE_URI + "/threat_intel/alerts/status"; + public static final String TEST_CONNECTION_BASE_URI = PLUGINS_BASE_URI + "/connections/%s/test"; + public static final String TEST_S3_CONNECTION_URI = String.format(TEST_CONNECTION_BASE_URI, "s3"); public static final String CUSTOM_LOG_TYPE_URI = PLUGINS_BASE_URI + "/logtype"; public static final String CORRELATIONS_ALERTS_BASE_URI = PLUGINS_BASE_URI + "/correlationAlerts"; public static final String JOB_INDEX_NAME = ".opensearch-sap--job"; + public static final String JOB_TYPE = "opensearch_sap_job"; + public static final Map TIF_JOB_INDEX_SETTING = Map.of(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1, IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-all", IndexMetadata.SETTING_INDEX_HIDDEN, true); private CorrelationRuleIndices correlationRuleIndices; @@ -159,13 +280,19 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map private BuiltinLogTypeLoader builtinLogTypeLoader; private LogTypeService logTypeService; + + private SATIFSourceConfigService saTifSourceConfigService; + @Override - public Collection getSystemIndexDescriptors(Settings settings){ - return Collections.singletonList(new SystemIndexDescriptor(THREAT_INTEL_DATA_INDEX_NAME_PREFIX, "System index used for threat intel data")); + public Collection getSystemIndexDescriptors(Settings settings) { + List descriptors = List.of( + new SystemIndexDescriptor(THREAT_INTEL_DATA_INDEX_NAME_PREFIX, "System index used for threat intel data"), + new SystemIndexDescriptor(CORRELATION_ALERT_INDEX, "System index used for Correlation Alerts") + ); + return descriptors; } - @Override public Collection createComponents(Client client, ClusterService clusterService, @@ -195,14 +322,24 @@ public Collection createComponents(Client client, TIFJobParameterService tifJobParameterService = new TIFJobParameterService(client, clusterService); TIFJobUpdateService tifJobUpdateService = new TIFJobUpdateService(clusterService, tifJobParameterService, threatIntelFeedDataService, builtInTIFMetadataLoader); TIFLockService threatIntelLockService = new TIFLockService(clusterService, client); + saTifSourceConfigService = new SATIFSourceConfigService(client, clusterService, threadPool, xContentRegistry, threatIntelLockService); + STIX2IOCFetchService stix2IOCFetchService = new STIX2IOCFetchService(client, clusterService); + SATIFSourceConfigManagementService saTifSourceConfigManagementService = new SATIFSourceConfigManagementService(saTifSourceConfigService, threatIntelLockService, stix2IOCFetchService, xContentRegistry, clusterService); + SecurityAnalyticsRunner.getJobRunnerInstance(); + TIFSourceConfigRunner.getJobRunnerInstance().initialize(clusterService, threatIntelLockService, threadPool, saTifSourceConfigManagementService, saTifSourceConfigService); CorrelationAlertService correlationAlertService = new CorrelationAlertService(client, xContentRegistry); - NotificationService notificationServiceService = new NotificationService((NodeClient)client, scriptService); + NotificationService notificationService = new NotificationService((NodeClient) client, scriptService); TIFJobRunner.getJobRunnerInstance().initialize(clusterService, tifJobUpdateService, tifJobParameterService, threatIntelLockService, threadPool, detectorThreatIntelService); - + IocFindingService iocFindingService = new IocFindingService(client, clusterService, xContentRegistry); + ThreatIntelAlertService threatIntelAlertService = new ThreatIntelAlertService(client, clusterService, xContentRegistry); + SaIoCScanService ioCScanService = new SaIoCScanService(client, clusterService, xContentRegistry, iocFindingService, threatIntelAlertService, notificationService); + DefaultTifSourceConfigLoaderService defaultTifSourceConfigLoaderService = new DefaultTifSourceConfigLoaderService(builtInTIFMetadataLoader, client, saTifSourceConfigManagementService); return List.of( - detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, + detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, threatIntelAlertService, mapperService, indexTemplateManager, builtinLogTypeLoader, builtInTIFMetadataLoader, threatIntelFeedDataService, detectorThreatIntelService, - tifJobUpdateService, tifJobParameterService, threatIntelLockService, correlationAlertService, notificationServiceService); + correlationAlertService, notificationService, + tifJobUpdateService, tifJobParameterService, threatIntelLockService, saTifSourceConfigService, saTifSourceConfigManagementService, stix2IOCFetchService, + ioCScanService, defaultTifSourceConfigLoaderService); } @Override @@ -230,6 +367,8 @@ public List getRestHandlers(Settings settings, new RestGetFindingsAction(), new RestGetMappingsViewAction(), new RestGetAlertsAction(), + new RestGetThreatIntelAlertsAction(), + new RestUpdateThreatIntelAlertsStatusAction(), new RestIndexRuleAction(), new RestSearchRuleAction(), new RestDeleteRuleAction(), @@ -243,6 +382,17 @@ public List getRestHandlers(Settings settings, new RestIndexCustomLogTypeAction(), new RestSearchCustomLogTypeAction(), new RestDeleteCustomLogTypeAction(), + new RestIndexTIFSourceConfigAction(), + new RestGetTIFSourceConfigAction(), + new RestDeleteTIFSourceConfigAction(), + new RestSearchTIFSourceConfigsAction(), + new RestIndexThreatIntelMonitorAction(), + new RestDeleteThreatIntelMonitorAction(), + new RestSearchThreatIntelMonitorAction(), + new RestRefreshTIFSourceConfigAction(), + new RestListIOCsAction(), + new RestGetIocFindingsAction(), + new RestTestS3ConnectionAction(), new RestGetCorrelationsAlertsAction(), new RestAcknowledgeCorrelationAlertsAction() ); @@ -250,7 +400,7 @@ public List getRestHandlers(Settings settings, @Override public String getJobType() { - return "opensearch_sap_job"; + return JOB_TYPE; } @Override @@ -260,12 +410,29 @@ public String getJobIndex() { @Override public ScheduledJobRunner getJobRunner() { - return TIFJobRunner.getJobRunnerInstance(); + return SecurityAnalyticsRunner.getJobRunnerInstance(); } @Override public ScheduledJobParser getJobParser() { - return (parser, id, jobDocVersion) -> TIFJobParameter.PARSER.parse(parser, null); + return (xcp, id, jobDocVersion) -> { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + if (xcp.nextToken() == XContentParser.Token.START_OBJECT) { + switch (fieldName) { + case SOURCE_CONFIG_FIELD: + return SATIFSourceConfig.parse(xcp, id, null); + default: + log.error("Job parser failed for [{}] in security analytics job registration", fieldName); + xcp.skipChildren(); + } + } else { + return TIFJobParameter.parseFromParser(xcp, id, jobDocVersion.getVersion()); + } + } + return null; + }; } @Override @@ -326,6 +493,11 @@ public List> getSettings() { SecurityAnalyticsSettings.CORRELATION_HISTORY_INDEX_MAX_AGE, SecurityAnalyticsSettings.CORRELATION_HISTORY_ROLLOVER_PERIOD, SecurityAnalyticsSettings.CORRELATION_HISTORY_RETENTION_PERIOD, + SecurityAnalyticsSettings.IOC_FINDING_HISTORY_ENABLED, + SecurityAnalyticsSettings.IOC_FINDING_HISTORY_MAX_DOCS, + SecurityAnalyticsSettings.IOC_FINDING_HISTORY_INDEX_MAX_AGE, + SecurityAnalyticsSettings.IOC_FINDING_HISTORY_ROLLOVER_PERIOD, + SecurityAnalyticsSettings.IOC_FINDING_HISTORY_RETENTION_PERIOD, SecurityAnalyticsSettings.IS_CORRELATION_INDEX_SETTING, SecurityAnalyticsSettings.CORRELATION_TIME_WINDOW, SecurityAnalyticsSettings.ENABLE_AUTO_CORRELATIONS, @@ -333,7 +505,11 @@ public List> getSettings() { SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE, SecurityAnalyticsSettings.TIF_UPDATE_INTERVAL, SecurityAnalyticsSettings.BATCH_SIZE, - SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT + SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT, + SecurityAnalyticsSettings.IOC_INDEX_RETENTION_PERIOD, + SecurityAnalyticsSettings.IOC_MAX_INDICES_PER_INDEX_PATTERN, + SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT, + SecurityAnalyticsSettings.ENABLE_DETECTORS_WITH_DEDICATED_QUERY_INDICES ); } @@ -362,9 +538,23 @@ public List> getSettings() { new ActionPlugin.ActionHandler<>(AlertingActions.SUBSCRIBE_FINDINGS_ACTION_TYPE, TransportCorrelateFindingAction.class), new ActionPlugin.ActionHandler<>(ListCorrelationsAction.INSTANCE, TransportListCorrelationAction.class), new ActionPlugin.ActionHandler<>(SearchCorrelationRuleAction.INSTANCE, TransportSearchCorrelationRuleAction.class), + new ActionPlugin.ActionHandler<>(GetThreatIntelAlertsAction.INSTANCE, TransportGetThreatIntelAlertsAction.class), + new ActionPlugin.ActionHandler<>(UpdateThreatIntelAlertStatusAction.INSTANCE, TransportUpdateThreatIntelAlertStatusAction.class), new ActionHandler<>(IndexCustomLogTypeAction.INSTANCE, TransportIndexCustomLogTypeAction.class), new ActionHandler<>(SearchCustomLogTypeAction.INSTANCE, TransportSearchCustomLogTypeAction.class), new ActionHandler<>(DeleteCustomLogTypeAction.INSTANCE, TransportDeleteCustomLogTypeAction.class), + new ActionHandler<>(IndexThreatIntelMonitorAction.INSTANCE, TransportIndexThreatIntelMonitorAction.class), + new ActionHandler<>(DeleteThreatIntelMonitorAction.INSTANCE, TransportDeleteThreatIntelMonitorAction.class), + new ActionHandler<>(SearchThreatIntelMonitorAction.INSTANCE, TransportSearchThreatIntelMonitorAction.class), + new ActionHandler<>(SAIndexTIFSourceConfigAction.INSTANCE, TransportIndexTIFSourceConfigAction.class), + new ActionHandler<>(SAGetTIFSourceConfigAction.INSTANCE, TransportGetTIFSourceConfigAction.class), + new ActionHandler<>(SADeleteTIFSourceConfigAction.INSTANCE, TransportDeleteTIFSourceConfigAction.class), + new ActionHandler<>(SASearchTIFSourceConfigsAction.INSTANCE, TransportSearchTIFSourceConfigsAction.class), + new ActionHandler<>(SARefreshTIFSourceConfigAction.INSTANCE, TransportRefreshTIFSourceConfigAction.class), + new ActionHandler<>(ThreatIntelMonitorRunner.REMOTE_DOC_LEVEL_MONITOR_ACTION_INSTANCE, TransportThreatIntelMonitorFanOutAction.class), + new ActionHandler<>(ListIOCsAction.INSTANCE, TransportListIOCsAction.class), + new ActionHandler<>(TestS3ConnectionAction.INSTANCE, TransportTestS3ConnectionAction.class), + new ActionHandler<>(GetIocFindingsAction.INSTANCE, TransportGetIocFindingsAction.class), new ActionHandler<>(PutTIFJobAction.INSTANCE, TransportPutTIFJobAction.class), new ActionPlugin.ActionHandler<>(GetCorrelationAlertsAction.INSTANCE, TransportGetCorrelationAlertsAction.class), new ActionPlugin.ActionHandler<>(AckCorrelationAlertsAction.INSTANCE, TransportAckCorrelationAlertsAction.class) @@ -386,4 +576,12 @@ public void onFailure(Exception e) { } }); } + + @NonNull + @Override + public Map getMonitorTypesToMonitorRunners() { + return Map.of( + THREAT_INTEL_MONITOR_TYPE, ThreatIntelMonitorRunner.getMonitorRunner() + ); + } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/action/GetAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/action/GetAlertsAction.java index df9422a77..39f415e90 100644 --- a/src/main/java/org/opensearch/securityanalytics/action/GetAlertsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/action/GetAlertsAction.java @@ -14,4 +14,4 @@ public class GetAlertsAction extends ActionType { public GetAlertsAction() { super(NAME, GetAlertsResponse::new); } -} \ No newline at end of file +} diff --git a/src/main/java/org/opensearch/securityanalytics/action/GetAlertsRequest.java b/src/main/java/org/opensearch/securityanalytics/action/GetAlertsRequest.java index 1e0cb6113..4f2009459 100644 --- a/src/main/java/org/opensearch/securityanalytics/action/GetAlertsRequest.java +++ b/src/main/java/org/opensearch/securityanalytics/action/GetAlertsRequest.java @@ -5,6 +5,7 @@ package org.opensearch.securityanalytics.action; import java.io.IOException; +import java.time.Instant; import java.util.Locale; import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; @@ -24,6 +25,10 @@ public class GetAlertsRequest extends ActionRequest { private String severityLevel; private String alertState; + private Instant startTime; + + private Instant endTime; + public static final String DETECTOR_ID = "detector_id"; public GetAlertsRequest( @@ -31,7 +36,9 @@ public GetAlertsRequest( String logType, Table table, String severityLevel, - String alertState + String alertState, + Instant startTime, + Instant endTime ) { super(); this.detectorId = detectorId; @@ -39,6 +46,8 @@ public GetAlertsRequest( this.table = table; this.severityLevel = severityLevel; this.alertState = alertState; + this.startTime = startTime; + this.endTime = endTime; } public GetAlertsRequest(StreamInput sin) throws IOException { this( @@ -46,7 +55,9 @@ public GetAlertsRequest(StreamInput sin) throws IOException { sin.readOptionalString(), Table.readFrom(sin), sin.readString(), - sin.readString() + sin.readString(), + sin.readOptionalInstant(), + sin.readOptionalInstant() ); } @@ -68,6 +79,8 @@ public void writeTo(StreamOutput out) throws IOException { table.writeTo(out); out.writeString(severityLevel); out.writeString(alertState); + out.writeOptionalInstant(startTime); + out.writeOptionalInstant(endTime); } public String getDetectorId() { @@ -89,4 +102,12 @@ public String getAlertState() { public String getLogType() { return logType; } + + public Instant getStartTime() { + return startTime; + } + + public Instant getEndTime() { + return endTime; + } } diff --git a/src/main/java/org/opensearch/securityanalytics/action/GetDetectorResponse.java b/src/main/java/org/opensearch/securityanalytics/action/GetDetectorResponse.java index 0d700b88c..955db1455 100644 --- a/src/main/java/org/opensearch/securityanalytics/action/GetDetectorResponse.java +++ b/src/main/java/org/opensearch/securityanalytics/action/GetDetectorResponse.java @@ -69,6 +69,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws .field(Detector.LAST_UPDATE_TIME_FIELD, detector.getLastUpdateTime()) .field(Detector.ENABLED_TIME_FIELD, detector.getEnabledTime()) .field(Detector.THREAT_INTEL_ENABLED_FIELD, detector.getThreatIntelEnabled()) + .field(Detector.TRIGGERS_FIELD, detector.getTriggers()) .endObject(); return builder.endObject(); } diff --git a/src/main/java/org/opensearch/securityanalytics/action/TestS3ConnectionAction.java b/src/main/java/org/opensearch/securityanalytics/action/TestS3ConnectionAction.java new file mode 100644 index 000000000..cb4d39421 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/action/TestS3ConnectionAction.java @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.action; + +import org.opensearch.action.ActionType; + +public class TestS3ConnectionAction extends ActionType { + public static final String NAME = "cluster:admin/opensearch/securityanalytics/connections/test/s3"; + public static final TestS3ConnectionAction INSTANCE = new TestS3ConnectionAction(); + + public TestS3ConnectionAction() { + super(NAME, TestS3ConnectionResponse::new); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/action/TestS3ConnectionRequest.java b/src/main/java/org/opensearch/securityanalytics/action/TestS3ConnectionRequest.java new file mode 100644 index 000000000..e69b0dde7 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/action/TestS3ConnectionRequest.java @@ -0,0 +1,80 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.ValidateActions; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.securityanalytics.commons.connector.model.S3ConnectorConfig; +import org.opensearch.securityanalytics.threatIntel.model.S3Source; + +import java.io.IOException; + +public class TestS3ConnectionRequest extends ActionRequest implements ToXContentObject { + private final S3Source s3Source; + + public TestS3ConnectionRequest(S3Source s3Source) { + super(); + this.s3Source = s3Source; + } + + public TestS3ConnectionRequest(String bucketName, String objectKey, String region, String roleArn) { + this(new S3Source(bucketName, objectKey, region, roleArn)); + } + + public TestS3ConnectionRequest(StreamInput sin) throws IOException { + this(new S3Source(sin)); + } + + public void writeTo(StreamOutput out) throws IOException { + s3Source.writeTo(out); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (s3Source.getBucketName() == null || s3Source.getBucketName().isEmpty()) { + validationException = ValidateActions.addValidationError("Must provide bucket name.", validationException); + } + if (s3Source.getObjectKey() == null || s3Source.getObjectKey().isEmpty()) { + validationException = ValidateActions.addValidationError("Must provide object key.", validationException); + } + if (s3Source.getObjectKey() == null || s3Source.getObjectKey().isEmpty()) { + validationException = ValidateActions.addValidationError("Must provide region.", validationException); + } + if (s3Source.getRoleArn() == null || s3Source.getRoleArn().isEmpty()) { + validationException = ValidateActions.addValidationError("Must provide role ARN.", validationException); + } + return validationException; + } + + public static TestS3ConnectionRequest parse(XContentParser xcp) throws IOException { + return new TestS3ConnectionRequest(S3Source.parse(xcp)); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return s3Source.toXContent(builder, params); + } + + public S3ConnectorConfig constructS3ConnectorConfig() { + return new S3ConnectorConfig( + s3Source.getBucketName(), + s3Source.getObjectKey(), + s3Source.getRegion(), + s3Source.getRoleArn() + ); + } + + public S3Source getS3Source() { + return s3Source; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/action/TestS3ConnectionResponse.java b/src/main/java/org/opensearch/securityanalytics/action/TestS3ConnectionResponse.java new file mode 100644 index 000000000..9e2bee5fd --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/action/TestS3ConnectionResponse.java @@ -0,0 +1,55 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.action; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; + +public class TestS3ConnectionResponse extends ActionResponse implements ToXContentObject { + public static final String STATUS_FIELD = "status"; + public static final String ERROR_FIELD = "error"; + + private RestStatus status; + private String error; + + public TestS3ConnectionResponse(RestStatus status, String error) { + super(); + this.status = status; + this.error = error; + } + + public TestS3ConnectionResponse(StreamInput sin) throws IOException { + this(sin.readEnum(RestStatus.class), sin.readOptionalString()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeEnum(status); + out.writeOptionalString(error); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field(STATUS_FIELD, status) + .field(ERROR_FIELD, error) + .endObject(); + } + + public RestStatus getStatus() { + return status; + } + + public String getError() { + return error; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java b/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java index 69757bf81..fa41aa983 100644 --- a/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java +++ b/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java @@ -19,6 +19,9 @@ import org.opensearch.commons.alerting.model.Alert; import org.opensearch.commons.alerting.model.Table; import org.opensearch.core.rest.RestStatus; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; import org.opensearch.securityanalytics.action.AckAlertsResponse; import org.opensearch.securityanalytics.action.AlertDto; import org.opensearch.securityanalytics.action.GetAlertsResponse; @@ -29,6 +32,7 @@ import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -36,6 +40,8 @@ import java.util.Map; import java.util.stream.Collectors; +import static org.opensearch.securityanalytics.transport.TransportIndexDetectorAction.CHAINED_FINDINGS_MONITOR_STRING; + /** * Alerts Service implements operations involving interaction with Alerting Plugin */ @@ -66,6 +72,8 @@ public void getAlertsByDetectorId( Table table, String severityLevel, String alertState, + Instant startTime, + Instant endTime, ActionListener listener ) { this.client.execute(GetDetectorAction.INSTANCE, new GetDetectorRequest(detectorId, -3L), new ActionListener<>() { @@ -78,16 +86,27 @@ public void onResponse(GetDetectorResponse getDetectorResponse) { // monitor --> detectorId mapping Map monitorToDetectorMapping = new HashMap<>(); detector.getMonitorIds().forEach( - monitorId -> monitorToDetectorMapping.put(monitorId, detector.getId()) + monitorId -> { + if (detector.getRuleIdMonitorIdMap().containsKey(CHAINED_FINDINGS_MONITOR_STRING)) { + if (detector.getRuleIdMonitorIdMap().get(CHAINED_FINDINGS_MONITOR_STRING).equals(monitorId) || + (detector.getRuleIdMonitorIdMap().containsKey("-1") && detector.getRuleIdMonitorIdMap().get("-1").equals(monitorId))) { + monitorToDetectorMapping.put(monitorId, detector.getId()); + } + } else { + monitorToDetectorMapping.put(monitorId, detector.getId()); + } + } ); // Get alerts for all monitor ids AlertsService.this.getAlertsByMonitorIds( monitorToDetectorMapping, - monitorIds, + new ArrayList<>(monitorToDetectorMapping.keySet()), DetectorMonitorConfig.getAllAlertsIndicesPattern(detector.getDetectorType()), table, severityLevel, alertState, + startTime, + endTime, new ActionListener<>() { @Override public void onResponse(GetAlertsResponse getAlertsResponse) { @@ -129,9 +148,11 @@ public void getAlertsByMonitorIds( Table table, String severityLevel, String alertState, + Instant startTime, + Instant endTime, ActionListener listener ) { - + BoolQueryBuilder boolQueryBuilder = getBoolQueryBuilder(startTime, endTime); org.opensearch.commons.alerting.action.GetAlertsRequest req = new org.opensearch.commons.alerting.action.GetAlertsRequest( table, @@ -142,7 +163,7 @@ public void getAlertsByMonitorIds( monitorIds, null, null, - null + boolQueryBuilder ); AlertingPluginInterface.INSTANCE.getAlerts((NodeClient) client, req, new ActionListener<>() { @@ -179,6 +200,8 @@ public void getAlerts( Table table, String severityLevel, String alertState, + Instant startTime, + Instant endTime, ActionListener listener ) { if (detectors.size() == 0) { @@ -205,6 +228,8 @@ public void getAlerts( table, severityLevel, alertState, + startTime, + endTime, new ActionListener<>() { @Override public void onResponse(GetAlertsResponse getAlertsResponse) { @@ -247,7 +272,10 @@ private AlertDto mapAlertToAlertDto(Alert alert, String detectorId) { public void getAlerts(List alertIds, Detector detector, Table table, + Instant startTime, + Instant endTime, ActionListener actionListener) { + BoolQueryBuilder boolQueryBuilder = getBoolQueryBuilder(startTime, endTime); GetAlertsRequest request = new GetAlertsRequest( table, "ALL", @@ -257,7 +285,7 @@ public void getAlerts(List alertIds, null, null, alertIds, - null); + boolQueryBuilder); AlertingPluginInterface.INSTANCE.getAlerts( (NodeClient) client, request, actionListener); @@ -307,4 +335,17 @@ public void onFailure(Exception e) { } } + + private static BoolQueryBuilder getBoolQueryBuilder(Instant startTime, Instant endTime) { + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + if (startTime != null && endTime != null) { + long startTimeMillis = startTime.toEpochMilli(); + long endTimeMillis = endTime.toEpochMilli(); + QueryBuilder timeRangeQuery = QueryBuilders.rangeQuery("start_time") + .from(startTimeMillis) // Greater than or equal to start time + .to(endTimeMillis); // Less than or equal to end time + boolQueryBuilder.filter(timeRangeQuery); + } + return boolQueryBuilder; + } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java index 459f523b7..02f6595ec 100644 --- a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java @@ -5,6 +5,8 @@ package org.opensearch.securityanalytics.config.monitors; import java.util.List; +import java.util.Random; +import java.util.UUID; import java.util.stream.Collectors; import org.opensearch.common.inject.Inject; import org.opensearch.securityanalytics.logtype.LogTypeService; @@ -25,6 +27,10 @@ public static String getRuleIndex(String logType) { return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries", logType); } + public static String getRuleIndexOptimized(String logType) { + return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-optimized-%s", logType, UUID.randomUUID()); + } + public static String getAlertsIndex(String logType) { return String.format(Locale.getDefault(), ".opensearch-sap-%s-alerts", logType); } diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationAlertService.java b/src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationAlertService.java index 54c09d29a..e5c43698b 100644 --- a/src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationAlertService.java +++ b/src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationAlertService.java @@ -26,6 +26,7 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.TermsQueryBuilder; @@ -188,7 +189,11 @@ public void getCorrelationAlerts(String ruleId, Table tableProp, ActionListener< }, e -> { log.error("Search request to fetch correlation alerts failed", e); - listener.onFailure(e); + if (e instanceof IndexNotFoundException) { + listener.onResponse(new GetCorrelationAlertsResponse(Collections.emptyList(), 0)); + } else { + listener.onFailure(e); + } } )); } diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/alert/notifications/NotificationService.java b/src/main/java/org/opensearch/securityanalytics/correlation/alert/notifications/NotificationService.java index be141a1d9..d081fd00f 100644 --- a/src/main/java/org/opensearch/securityanalytics/correlation/alert/notifications/NotificationService.java +++ b/src/main/java/org/opensearch/securityanalytics/correlation/alert/notifications/NotificationService.java @@ -12,13 +12,14 @@ import org.opensearch.commons.notifications.model.ChannelMessage; import org.opensearch.commons.notifications.model.EventSource; import org.opensearch.commons.notifications.model.SeverityType; -import org.opensearch.commons.notifications.model.NotificationConfigInfo; import org.opensearch.commons.notifications.action.GetNotificationConfigRequest; import org.opensearch.commons.notifications.action.GetNotificationConfigResponse; import org.opensearch.core.action.ActionListener; import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelAlertContext; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import org.opensearch.script.ScriptService; + import java.io.IOException; import java.util.HashMap; import java.util.List; @@ -27,9 +28,9 @@ import java.util.Set; import java.util.HashSet; import java.util.Collections; + import org.opensearch.script.Script; import org.opensearch.script.TemplateScript; -import org.opensearch.commons.notifications.model.SeverityType; public class NotificationService { @@ -42,6 +43,7 @@ public NotificationService(NodeClient client, ScriptService scriptService) { this.client = client; this.scriptService = scriptService; } + /** * Extension function for publishing a notification to a channel in the Notification plugin. */ @@ -53,13 +55,11 @@ public void sendNotification(String configId, String severity, String subject, S NotificationsPluginInterface.INSTANCE.sendNotification(client, new EventSource(subject, configId, severityType, Collections.emptyList()), message, channelIds, new ActionListener() { @Override public void onResponse(SendNotificationResponse sendNotificationResponse) { - if(sendNotificationResponse.getStatus() == RestStatus.OK) { + if (sendNotificationResponse.getStatus() == RestStatus.OK) { logger.info("Successfully sent a notification, Notification Event: " + sendNotificationResponse.getNotificationEvent()); - } - else { + } else { logger.error("Error while sending a notification, Notification Event: " + sendNotificationResponse.getNotificationEvent()); } - } @Override public void onFailure(Exception e) { @@ -68,6 +68,31 @@ public void onFailure(Exception e) { }); } + /** + * Extension function for publishing a notification to a channel in the Notification plugin. + */ + public void sendNotification(String configId, String severity, String subject, String notificationMessageText, + ActionListener listener) { + ChannelMessage message = generateMessage(notificationMessageText); + List channelIds = new ArrayList<>(); + channelIds.add(configId); + SeverityType severityType = SeverityType.Companion.fromTagOrDefault(severity); + NotificationsPluginInterface.INSTANCE.sendNotification(client, new EventSource(subject, configId, severityType, Collections.emptyList()), message, channelIds, ActionListener.wrap( + sendNotificationResponse -> { + if (sendNotificationResponse.getStatus() == RestStatus.OK) { + logger.info("Successfully sent a notification, Notification Event: " + sendNotificationResponse.getNotificationEvent()); + listener.onResponse(null); + } else { + listener.onFailure(new Exception("Error while sending a notification, Notification Event: " + sendNotificationResponse.getNotificationEvent())); + } + + }, e -> { + logger.error("Failed while sending a notification with " + configId, e); + listener.onFailure(e); + } + )); + } + /** * Gets a NotificationConfigInfo object by ID if it exists. */ @@ -106,9 +131,17 @@ public static ChannelMessage generateMessage(String message) { } public static String compileTemplate(CorrelationAlertContext ctx, Script template) { + return compileTemplateGeneric(template, ctx.asTemplateArg()); + } + + public static String compileTemplate(ThreatIntelAlertContext ctx, Script template) { + return compileTemplateGeneric(template, ctx.asTemplateArg()); + } + + private static String compileTemplateGeneric(Script template, Map templateArg) { TemplateScript.Factory factory = scriptService.compile(template, TemplateScript.CONTEXT); Map params = new HashMap<>(template.getParams()); - params.put("ctx", ctx.asTemplateArg()); + params.put("ctx", templateArg); TemplateScript templateScript = factory.newInstance(params); return templateScript.execute(); } diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/CorrelationCodecVersion.java b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/CorrelationCodecVersion.java index 07721ae9b..7f4c84eb9 100644 --- a/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/CorrelationCodecVersion.java +++ b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/CorrelationCodecVersion.java @@ -4,11 +4,13 @@ */ package org.opensearch.securityanalytics.correlation.index.codec; +import org.apache.lucene.backward_codecs.lucene99.Lucene99Codec; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.backward_codecs.lucene95.Lucene95Codec; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.opensearch.index.mapper.MapperService; +import org.opensearch.securityanalytics.correlation.index.codec.correlation9120.CorrelationCodec9120; import org.opensearch.securityanalytics.correlation.index.codec.correlation950.CorrelationCodec950; import org.opensearch.securityanalytics.correlation.index.codec.correlation990.CorrelationCodec990; import org.opensearch.securityanalytics.correlation.index.codec.correlation990.PerFieldCorrelationVectorsFormat990; @@ -20,7 +22,7 @@ public enum CorrelationCodecVersion { V_9_5_0( - "CorrelationCodec950", + "CorrelationCodec", new Lucene95Codec(), new PerFieldCorrelationVectorsFormat950(Optional.empty()), (userCodec, mapperService) -> new CorrelationCodec950(userCodec, new PerFieldCorrelationVectorsFormat950(Optional.of(mapperService))), @@ -32,9 +34,16 @@ public enum CorrelationCodecVersion { new PerFieldCorrelationVectorsFormat990(Optional.empty()), (userCodec, mapperService) -> new CorrelationCodec990(userCodec, new PerFieldCorrelationVectorsFormat990(Optional.of(mapperService))), CorrelationCodec990::new + ), + V_9_12_0( + "CorrelationCodec9120", + new Lucene912Codec(), + new PerFieldCorrelationVectorsFormat990(Optional.empty()), + (userCodec, mapperService) -> new CorrelationCodec9120(userCodec, new PerFieldCorrelationVectorsFormat990(Optional.of(mapperService))), + CorrelationCodec9120::new ); - private static final CorrelationCodecVersion CURRENT = V_9_9_0; + private static final CorrelationCodecVersion CURRENT = V_9_12_0; private final String codecName; private final Codec defaultCodecDelegate; private final PerFieldKnnVectorsFormat perFieldKnnVectorsFormat; diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation9120/CorrelationCodec9120.java b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation9120/CorrelationCodec9120.java new file mode 100644 index 000000000..1aa6e6824 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation9120/CorrelationCodec9120.java @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.correlation.index.codec.correlation9120; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.opensearch.securityanalytics.correlation.index.codec.CorrelationCodecVersion; + +public class CorrelationCodec9120 extends FilterCodec { + private static final CorrelationCodecVersion VERSION = CorrelationCodecVersion.V_9_12_0; + private final PerFieldKnnVectorsFormat perFieldCorrelationVectorsFormat; + + public CorrelationCodec9120() { + this(VERSION.getDefaultCodecDelegate(), VERSION.getPerFieldCorrelationVectorsFormat()); + } + + public CorrelationCodec9120(Codec delegate, PerFieldKnnVectorsFormat perFieldCorrelationVectorsFormat) { + super(VERSION.getCodecName(), delegate); + this.perFieldCorrelationVectorsFormat = perFieldCorrelationVectorsFormat; + } + + @Override + public KnnVectorsFormat knnVectorsFormat() { + return perFieldCorrelationVectorsFormat; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java index f6630499f..22b415230 100644 --- a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java @@ -35,6 +35,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; import org.opensearch.securityanalytics.logtype.LogTypeService; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService; import org.opensearch.securityanalytics.util.CorrelationIndices; import org.opensearch.threadpool.Scheduler; import org.opensearch.threadpool.ThreadPool; @@ -54,9 +55,13 @@ public class DetectorIndexManagementService extends AbstractLifecycleComponent i private volatile Boolean alertHistoryEnabled; private volatile Boolean findingHistoryEnabled; + private volatile Boolean iocFindingHistoryEnabled; + private volatile Long alertHistoryMaxDocs; private volatile Long findingHistoryMaxDocs; + private volatile Long iocFindingHistoryMaxDocs; + private volatile Long correlationHistoryMaxDocs; private volatile TimeValue alertHistoryMaxAge; @@ -64,16 +69,22 @@ public class DetectorIndexManagementService extends AbstractLifecycleComponent i private volatile TimeValue correlationHistoryMaxAge; + private volatile TimeValue iocFindingHistoryMaxAge; + private volatile TimeValue alertHistoryRolloverPeriod; private volatile TimeValue findingHistoryRolloverPeriod; private volatile TimeValue correlationHistoryRolloverPeriod; + private volatile TimeValue iocFindingHistoryRolloverPeriod; + private volatile TimeValue alertHistoryRetentionPeriod; private volatile TimeValue findingHistoryRetentionPeriod; private volatile TimeValue correlationHistoryRetentionPeriod; + private volatile TimeValue iocFindingHistoryRetentionPeriod; + private volatile boolean isClusterManager = false; private Scheduler.Cancellable scheduledAlertsRollover = null; @@ -81,11 +92,15 @@ public class DetectorIndexManagementService extends AbstractLifecycleComponent i private Scheduler.Cancellable scheduledCorrelationHistoryRollover = null; + private Scheduler.Cancellable scheduledIocFindingHistoryRollover = null; + List alertHistoryIndices = new ArrayList<>(); List findingHistoryIndices = new ArrayList<>(); HistoryIndexInfo correlationHistoryIndex = null; + HistoryIndexInfo iocFindingHistoryIndex = null; + @Inject public DetectorIndexManagementService( Settings settings, @@ -161,6 +176,27 @@ public DetectorIndexManagementService( clusterService.getClusterSettings().addSettingsUpdateConsumer(CORRELATION_HISTORY_RETENTION_PERIOD, this::setCorrelationHistoryRetentionPeriod); + clusterService.getClusterSettings().addSettingsUpdateConsumer(IOC_FINDING_HISTORY_MAX_DOCS, maxDocs -> { + setIocFindingHistoryMaxDocs(maxDocs); + if (iocFindingHistoryIndex != null) { + iocFindingHistoryIndex.maxDocs = maxDocs; + } + }); + + clusterService.getClusterSettings().addSettingsUpdateConsumer(IOC_FINDING_HISTORY_INDEX_MAX_AGE, maxAge -> { + setIocFindingHistoryMaxAge(maxAge); + if (iocFindingHistoryIndex != null) { + iocFindingHistoryIndex.maxAge = maxAge; + } + }); + + clusterService.getClusterSettings().addSettingsUpdateConsumer(IOC_FINDING_HISTORY_ROLLOVER_PERIOD, timeValue -> { + DetectorIndexManagementService.this.iocFindingHistoryRolloverPeriod = timeValue; + rescheduleIocFindingHistoryRollover(); + }); + + clusterService.getClusterSettings().addSettingsUpdateConsumer(IOC_FINDING_HISTORY_RETENTION_PERIOD, this::setIocFindingHistoryRetentionPeriod); + initFromClusterSettings(); } @@ -204,15 +240,19 @@ private void initFromClusterSettings() { alertHistoryMaxDocs = ALERT_HISTORY_MAX_DOCS.get(settings); findingHistoryMaxDocs = FINDING_HISTORY_MAX_DOCS.get(settings); correlationHistoryMaxDocs = CORRELATION_HISTORY_MAX_DOCS.get(settings); + iocFindingHistoryMaxDocs = IOC_FINDING_HISTORY_MAX_DOCS.get(settings); alertHistoryMaxAge = ALERT_HISTORY_INDEX_MAX_AGE.get(settings); findingHistoryMaxAge = FINDING_HISTORY_INDEX_MAX_AGE.get(settings); correlationHistoryMaxAge = CORRELATION_HISTORY_INDEX_MAX_AGE.get(settings); + iocFindingHistoryMaxAge = IOC_FINDING_HISTORY_INDEX_MAX_AGE.get(settings); alertHistoryRolloverPeriod = ALERT_HISTORY_ROLLOVER_PERIOD.get(settings); findingHistoryRolloverPeriod = FINDING_HISTORY_ROLLOVER_PERIOD.get(settings); correlationHistoryRolloverPeriod = CORRELATION_HISTORY_ROLLOVER_PERIOD.get(settings); + iocFindingHistoryRolloverPeriod = IOC_FINDING_HISTORY_ROLLOVER_PERIOD.get(settings); alertHistoryRetentionPeriod = ALERT_HISTORY_RETENTION_PERIOD.get(settings); findingHistoryRetentionPeriod = FINDING_HISTORY_RETENTION_PERIOD.get(settings); correlationHistoryRetentionPeriod = CORRELATION_HISTORY_RETENTION_PERIOD.get(settings); + iocFindingHistoryRetentionPeriod = IOC_FINDING_HISTORY_RETENTION_PERIOD.get(settings); } @Override @@ -238,6 +278,9 @@ public void clusterChanged(ClusterChangedEvent event) { if (correlationHistoryIndex != null && correlationHistoryIndex.indexAlias != null) { correlationHistoryIndex.isInitialized = event.state().metadata().hasAlias(correlationHistoryIndex.indexAlias); } + if (iocFindingHistoryIndex != null && iocFindingHistoryIndex.indexAlias != null) { + iocFindingHistoryIndex.isInitialized = event.state().metadata().hasAlias(iocFindingHistoryIndex.indexAlias); + } } private void onMaster() { @@ -247,6 +290,7 @@ private void onMaster() { rolloverAndDeleteAlertHistoryIndices(); rolloverAndDeleteFindingHistoryIndices(); rolloverAndDeleteCorrelationHistoryIndices(); + rolloverAndDeleteIocFindingHistoryIndices(); }, TimeValue.timeValueSeconds(1), executorName()); // schedule the next rollover for approx MAX_AGE later scheduledAlertsRollover = threadPool @@ -255,11 +299,13 @@ private void onMaster() { .scheduleWithFixedDelay(() -> rolloverAndDeleteFindingHistoryIndices(), findingHistoryRolloverPeriod, executorName()); scheduledCorrelationHistoryRollover = threadPool .scheduleWithFixedDelay(() -> rolloverAndDeleteCorrelationHistoryIndices(), correlationHistoryRolloverPeriod, executorName()); + scheduledIocFindingHistoryRollover = threadPool + .scheduleWithFixedDelay(() -> rolloverAndDeleteIocFindingHistoryIndices(), iocFindingHistoryRolloverPeriod, executorName()); } catch (Exception e) { // This should be run on cluster startup logger.error( - "Error creating alert/finding/correlation indices. " + - "Alerts/Findings/Correlations can't be recorded until master node is restarted.", + "Error creating alert/finding/correlation/ioc finding indices. " + + "Alerts/Findings/Correlations/IOC Finding can't be recorded until master node is restarted.", e ); } @@ -275,6 +321,9 @@ private void offMaster() { if (scheduledCorrelationHistoryRollover != null) { scheduledCorrelationHistoryRollover.cancel(); } + if (scheduledIocFindingHistoryRollover != null) { + scheduledIocFindingHistoryRollover.cancel(); + } } private String executorName() { @@ -327,6 +376,10 @@ private List getIndicesToDelete(ClusterStateResponse clusterStateRespons if (indexToDelete != null) { indicesToDelete.add(indexToDelete); } + indexToDelete = getHistoryIndexToDelete(indexMetaData, iocFindingHistoryRetentionPeriod.millis(), iocFindingHistoryIndex != null? List.of(iocFindingHistoryIndex): List.of(), true); + if (indexToDelete != null) { + indicesToDelete.add(indexToDelete); + } } return indicesToDelete; } @@ -371,7 +424,7 @@ private void deleteAllOldHistoryIndices(List indicesToDelete) { public void onResponse(AcknowledgedResponse deleteIndicesResponse) { if (!deleteIndicesResponse.isAcknowledged()) { logger.error( - "Could not delete one or more Alerting/Finding/Correlation history indices: [" + indicesToDelete + "]. Retrying one by one." + "Could not delete one or more Alerting/Finding/Correlation/IOC Finding history indices: [" + indicesToDelete + "]. Retrying one by one." ); deleteOldHistoryIndex(indicesToDelete); } else { @@ -381,7 +434,7 @@ public void onResponse(AcknowledgedResponse deleteIndicesResponse) { @Override public void onFailure(Exception e) { - logger.error("Delete for Alerting/Finding/Correlation History Indices failed: [" + indicesToDelete + "]. Retrying one By one."); + logger.error("Delete for Alerting/Finding/Correlation/IOC Finding History Indices failed: [" + indicesToDelete + "]. Retrying one By one."); deleteOldHistoryIndex(indicesToDelete); } } @@ -399,7 +452,7 @@ private void deleteOldHistoryIndex(List indicesToDelete) { @Override public void onResponse(AcknowledgedResponse acknowledgedResponse) { if (!acknowledgedResponse.isAcknowledged()) { - logger.error("Could not delete one or more Alerting/Finding/Correlation history indices: " + index); + logger.error("Could not delete one or more Alerting/Finding/Correlation/IOC Finding history indices: " + index); } } @@ -455,6 +508,23 @@ private void rolloverAndDeleteCorrelationHistoryIndices() { } } + private void rolloverAndDeleteIocFindingHistoryIndices() { + try { + iocFindingHistoryIndex = new HistoryIndexInfo( + IocFindingService.IOC_FINDING_ALIAS_NAME, + IocFindingService.IOC_FINDING_INDEX_PATTERN, + IocFindingService.getIndexMapping(), + iocFindingHistoryMaxDocs, + iocFindingHistoryMaxAge, + clusterService.state().metadata().hasAlias(IocFindingService.IOC_FINDING_ALIAS_NAME) + ); + rolloverIocFindingHistoryIndices(); + deleteOldIndices("IOC Findings", IocFindingService.IOC_FINDING_INDEX_PATTERN_REGEXP); + } catch (Exception ex) { + logger.error("failed to construct ioc finding index info"); + } + } + private List getAllAlertsIndicesPatternForAllTypes(List logTypes) { return logTypes .stream() @@ -488,8 +558,17 @@ private void rolloverIndex( request.getCreateIndexRequest().index(pattern) .mapping(map) .settings(isCorrelation? - Settings.builder().put("index.hidden", true).put("index.correlation", true).build(): - Settings.builder().put("index.hidden", true).build() + Settings.builder() + .put("index.hidden", true) + .put("index.correlation", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) + .build(): + Settings.builder() + .put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) + .build() ); request.addMaxIndexDocsCondition(docsCondition); request.addMaxIndexAgeCondition(ageCondition); @@ -544,6 +623,20 @@ private void rolloverCorrelationHistoryIndices() { } } + private void rolloverIocFindingHistoryIndices() { + if (iocFindingHistoryIndex != null) { + rolloverIndex( + iocFindingHistoryIndex.isInitialized, + iocFindingHistoryIndex.indexAlias, + iocFindingHistoryIndex.indexPattern, + iocFindingHistoryIndex.indexMappings, + iocFindingHistoryIndex.maxDocs, + iocFindingHistoryIndex.maxAge, + true + ); + } + } + private void rescheduleAlertRollover() { if (clusterService.state().getNodes().isLocalNodeElectedClusterManager()) { if (scheduledAlertsRollover != null) { @@ -574,6 +667,16 @@ private void rescheduleCorrelationHistoryRollover() { } } + private void rescheduleIocFindingHistoryRollover() { + if (clusterService.state().getNodes().isLocalNodeElectedClusterManager()) { + if (scheduledIocFindingHistoryRollover != null) { + scheduledIocFindingHistoryRollover.cancel(); + } + scheduledIocFindingHistoryRollover = threadPool + .scheduleWithFixedDelay(() -> rolloverAndDeleteIocFindingHistoryIndices(), iocFindingHistoryRolloverPeriod, executorName()); + } + } + private String alertMapping() { String alertMapping = null; try ( @@ -620,6 +723,10 @@ public void setCorrelationHistoryMaxDocs(Long correlationHistoryMaxDocs) { this.correlationHistoryMaxDocs = correlationHistoryMaxDocs; } + public void setIocFindingHistoryMaxDocs(Long iocFindingHistoryMaxDocs) { + this.iocFindingHistoryMaxDocs = iocFindingHistoryMaxDocs; + } + public void setAlertHistoryMaxAge(TimeValue alertHistoryMaxAge) { this.alertHistoryMaxAge = alertHistoryMaxAge; } @@ -632,6 +739,10 @@ public void setCorrelationHistoryMaxAge(TimeValue correlationHistoryMaxAge) { this.correlationHistoryMaxAge = correlationHistoryMaxAge; } + public void setIocFindingHistoryMaxAge(TimeValue iocFindingHistoryMaxAge) { + this.iocFindingHistoryMaxAge = iocFindingHistoryMaxAge; + } + public void setAlertHistoryRolloverPeriod(TimeValue alertHistoryRolloverPeriod) { this.alertHistoryRolloverPeriod = alertHistoryRolloverPeriod; } @@ -656,6 +767,10 @@ public void setCorrelationHistoryRetentionPeriod(TimeValue correlationHistoryRet this.correlationHistoryRetentionPeriod = correlationHistoryRetentionPeriod; } + public void setIocFindingHistoryRetentionPeriod(TimeValue iocFindingHistoryRetentionPeriod) { + this.iocFindingHistoryRetentionPeriod = iocFindingHistoryRetentionPeriod; + } + public void setClusterManager(boolean clusterManager) { isClusterManager = clusterManager; } @@ -676,6 +791,9 @@ protected void doStop() { if (scheduledCorrelationHistoryRollover != null) { scheduledCorrelationHistoryRollover.cancel(); } + if (scheduledIocFindingHistoryRollover != null) { + scheduledIocFindingHistoryRollover.cancel(); + } } @Override @@ -689,6 +807,9 @@ protected void doClose() { if (scheduledCorrelationHistoryRollover != null) { scheduledCorrelationHistoryRollover.cancel(); } + if (scheduledIocFindingHistoryRollover != null) { + scheduledIocFindingHistoryRollover.cancel(); + } } private static class HistoryIndexInfo { diff --git a/src/main/java/org/opensearch/securityanalytics/jobscheduler/SecurityAnalyticsRunner.java b/src/main/java/org/opensearch/securityanalytics/jobscheduler/SecurityAnalyticsRunner.java new file mode 100644 index 000000000..44ea1971d --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/jobscheduler/SecurityAnalyticsRunner.java @@ -0,0 +1,43 @@ +package org.opensearch.securityanalytics.jobscheduler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.jobscheduler.spi.JobExecutionContext; +import org.opensearch.jobscheduler.spi.ScheduledJobParameter; +import org.opensearch.jobscheduler.spi.ScheduledJobRunner; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobRunner; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFSourceConfigRunner; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.sacommons.TIFSourceConfig; + +public class SecurityAnalyticsRunner implements ScheduledJobRunner { + private static final Logger log = LogManager.getLogger(SecurityAnalyticsRunner.class); + + private static SecurityAnalyticsRunner INSTANCE; + public static SecurityAnalyticsRunner getJobRunnerInstance() { + if (INSTANCE != null) { + return INSTANCE; + } + synchronized (SecurityAnalyticsRunner.class) { + if (INSTANCE != null) { + return INSTANCE; + } + INSTANCE = new SecurityAnalyticsRunner(); + return INSTANCE; + } + } + private SecurityAnalyticsRunner() {} + + @Override + public void runJob(ScheduledJobParameter job, JobExecutionContext context) { + if (job instanceof TIFSourceConfig) { + TIFSourceConfigRunner.getJobRunnerInstance().runJob(job, context); + } else if (job instanceof TIFJobParameter) { + TIFJobRunner.getJobRunnerInstance().runJob(job, context); + } else { + String errorMessage = "Invalid job type, found " + job.getClass().getSimpleName() + "with id: " + context.getJobId(); + log.error(errorMessage); + throw new IllegalArgumentException(errorMessage); + } + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java b/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java index 0d28bce4d..2e77b0831 100644 --- a/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java +++ b/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java @@ -10,6 +10,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -70,7 +71,6 @@ private List loadBuiltinLogTypes() throws URISyntaxException, IOExcepti List logTypes = new ArrayList<>(); final String url = Objects.requireNonNull(BuiltinLogTypeLoader.class.getClassLoader().getResource(BASE_PATH)).toURI().toString(); - Path dirPath = null; if (url.contains("!")) { final String[] paths = url.split("!"); diff --git a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java index bc4452650..f42e59b78 100644 --- a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java +++ b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java @@ -63,10 +63,10 @@ import org.opensearch.securityanalytics.model.LogType; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; -import static org.opensearch.action.support.ActiveShardCount.ALL; import static org.opensearch.securityanalytics.model.FieldMappingDoc.LOG_TYPES; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.DEFAULT_MAPPING_SCHEMA; - +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; /** * @@ -459,7 +459,8 @@ public void ensureConfigIndexIsInitialized(ActionListener listener) { isConfigIndexInitialized = false; Settings indexSettings = Settings.builder() .put("index.hidden", true) - .put("index.auto_expand_replicas", "0-all") + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) .build(); CreateIndexRequest createIndexRequest = new CreateIndexRequest(); @@ -565,6 +566,9 @@ private List createFieldMappingDocs(List logTypes) { if (mapping.getOcsf() != null) { schemaFields.put("ocsf", mapping.getOcsf()); } + if (mapping.getOcsf11() != null) { + schemaFields.put("ocsf11", mapping.getOcsf11()); + } fieldMappingMap.put( key, new FieldMappingDoc( @@ -576,6 +580,7 @@ private List createFieldMappingDocs(List logTypes) { } else { // merge with existing doc existingDoc.getSchemaFields().put("ocsf", mapping.getOcsf()); + existingDoc.getSchemaFields().put("ocsf11", mapping.getOcsf11()); existingDoc.getLogTypes().add(logType.getName()); } })); @@ -704,7 +709,7 @@ public void getRuleFieldMappingsAllSchemas(String logType, ActionListener { List ruleFieldMappings = new ArrayList<>(); fieldMappingDocs.forEach( e -> { - ruleFieldMappings.add(new LogType.Mapping(e.getRawField(), e.getSchemaFields().get("ecs"), e.getSchemaFields().get("ocsf"))); + ruleFieldMappings.add(new LogType.Mapping(e.getRawField(), e.getSchemaFields().get("ecs"), e.getSchemaFields().get("ocsf"), e.getSchemaFields().get("ocsf11"))); }); delegatedListener.onResponse(ruleFieldMappings); } @@ -727,7 +732,8 @@ public void getRequiredFields(String logType, ActionListener { - // Find template which matches input index best + // Find template which matches input index best. starts by directly matching with input index and + // if not found matches with current write index. String templateName = MetadataIndexTemplateService.findV2Template( state.metadata(), normalizeIndexName(indexName), false ); + if (templateName == null) { + templateName = + MetadataIndexTemplateService.findV2Template( + state.metadata(), + normalizeIndexName(cin), + false + ); + } if (templateName == null) { // If we find conflicting templates(regardless of priority) and that template was created by us, @@ -181,8 +190,8 @@ public void upsertIndexTemplateWithAliasMappings( template = state.metadata().templatesV2().get(templateName); if (template.composedOf().contains(componentName) == false) { List newComposedOf = new ArrayList<>(template.composedOf()); - List indexPatterns = List.of(computeIndexPattern(indexName)); - ; + List indexPatterns = new ArrayList<>(template.indexPatterns()); + indexPatterns.add(computeIndexPattern(indexName)); newComposedOf.add(componentName); try { diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java index 42b374735..8b1e61349 100644 --- a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java +++ b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java @@ -232,6 +232,10 @@ public void onResponse(List mappings) { for (LogType.Mapping mapping : mappings) { if (indexFields.contains(mapping.getRawField())) { aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getRawField())); + } else if (indexFields.contains(mapping.getOcsf11())) { + // it's important to first check for OCSF1.1 before checking for OCSF1.0 + // changing this order leads to multiple ECS fields mapping to the same OCSF1.1 field + aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf11())); } else if (indexFields.contains(mapping.getOcsf())) { aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf())); } @@ -483,6 +487,7 @@ public void onResponse(GetMappingsResponse getMappingsResponse) { String alias = requiredField.getEcs(); String rawPath = requiredField.getRawField(); String ocsfPath = requiredField.getOcsf(); + String ocsf11Path = requiredField.getOcsf11(); if (allFieldsFromIndex.contains(rawPath)) { // if the alias was already added into applyable aliases, then skip to avoid duplicates if (!applyableAliases.contains(alias) && !applyableAliases.contains(rawPath)) { @@ -494,6 +499,9 @@ public void onResponse(GetMappingsResponse getMappingsResponse) { } pathsOfApplyableAliases.add(rawPath); } + } else if (allFieldsFromIndex.contains(ocsf11Path)) { + applyableAliases.add(alias); + pathsOfApplyableAliases.add(ocsf11Path); } else if (allFieldsFromIndex.contains(ocsfPath)) { applyableAliases.add(alias); pathsOfApplyableAliases.add(ocsfPath); @@ -518,7 +526,9 @@ public void onResponse(GetMappingsResponse getMappingsResponse) { Map> aliasMappingFields = new HashMap<>(); XContentBuilder aliasMappingsObj = XContentFactory.jsonBuilder().startObject(); for (LogType.Mapping mapping : requiredFields) { - if (allFieldsFromIndex.contains(mapping.getOcsf())) { + if (allFieldsFromIndex.contains(mapping.getOcsf11())) { + aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf11())); + } else if (allFieldsFromIndex.contains(mapping.getOcsf())) { aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf())); } else if (mapping.getEcs() != null) { shouldUpdateEcsMappingAndMaybeUpdates(mapping, aliasMappingFields, pathsOfApplyableAliases); diff --git a/src/main/java/org/opensearch/securityanalytics/model/DetailedSTIX2IOCDto.java b/src/main/java/org/opensearch/securityanalytics/model/DetailedSTIX2IOCDto.java new file mode 100644 index 000000000..cdcca8368 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/model/DetailedSTIX2IOCDto.java @@ -0,0 +1,209 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.model; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.commons.model.STIX2; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +/** + * A data transfer object for STIX2IOC containing additional details. + */ +public class DetailedSTIX2IOCDto implements Writeable, ToXContentObject { + public static final String NUM_FINDINGS_FIELD = "num_findings"; + STIX2IOCDto ioc; + private long numFindings = 0L; + + public DetailedSTIX2IOCDto( + STIX2IOCDto ioc, + long numFindings + ) { + this.ioc = ioc; + this.numFindings = numFindings; + } + + public DetailedSTIX2IOCDto(StreamInput sin) throws IOException { + this(STIX2IOCDto.readFrom(sin), sin.readLong()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + ioc.writeTo(out); + out.writeLong(numFindings); + } + + public static DetailedSTIX2IOCDto parse(XContentParser xcp, String id, Long version) throws IOException { + long numFindings = 0; + if (id == null) { + id = STIX2IOC.NO_ID; + } + + if (version == null) { + version = STIX2IOC.NO_VERSION; + } + + String name = null; + IOCType type = null; + String value = null; + String severity = null; + Instant created = null; + Instant modified = null; + String description = null; + List labels = new ArrayList<>(); + String specVersion = null; + String feedId = null; + String feedName = null; + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case STIX2.ID_FIELD: + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { + id = xcp.text(); + } + break; + case STIX2IOC.VERSION_FIELD: + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { + version = xcp.longValue(); + } + break; + case STIX2.NAME_FIELD: + name = xcp.text(); + break; + case STIX2.TYPE_FIELD: + type = new IOCType(xcp.text().toLowerCase(Locale.ROOT)); + break; + case STIX2.VALUE_FIELD: + value = xcp.text(); + break; + case STIX2.SEVERITY_FIELD: + severity = xcp.text(); + break; + case STIX2.CREATED_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + created = null; + } else if (xcp.currentToken().isValue()) { + if (xcp.currentToken() == XContentParser.Token.VALUE_STRING) { + created = Instant.parse(xcp.text()); + } else if (xcp.currentToken() == XContentParser.Token.VALUE_NUMBER) { + created = Instant.ofEpochMilli(xcp.longValue()); + } + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + created = null; + } + break; + case STIX2.MODIFIED_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + modified = null; + } else if (xcp.currentToken().isValue()) { + if (xcp.currentToken() == XContentParser.Token.VALUE_STRING) { + modified = Instant.parse(xcp.text()); + } else if (xcp.currentToken() == XContentParser.Token.VALUE_NUMBER) { + modified = Instant.ofEpochMilli(xcp.longValue()); + } + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + modified = null; + } + break; + case STIX2.DESCRIPTION_FIELD: + description = xcp.text(); + break; + case STIX2.LABELS_FIELD: + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + String entry = xcp.textOrNull(); + if (entry != null) { + labels.add(entry); + } + } + break; + case STIX2.SPEC_VERSION_FIELD: + specVersion = xcp.text(); + break; + case STIX2IOC.FEED_ID_FIELD: + feedId = xcp.text(); + break; + case STIX2IOC.FEED_NAME_FIELD: + feedName = xcp.text(); + break; + case NUM_FINDINGS_FIELD: + numFindings = xcp.longValue(); + break; + default: + xcp.skipChildren(); + } + } + + return new DetailedSTIX2IOCDto(new STIX2IOCDto( + id, + name, + type, + value, + severity, + created, + modified, + description, + labels, + specVersion, + feedId, + feedName, + version + ), numFindings); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field(STIX2IOC.ID_FIELD, ioc.getId()) + .field(STIX2IOC.NAME_FIELD, ioc.getName()) + .field(STIX2IOC.TYPE_FIELD, ioc.getType().toString()) + .field(STIX2IOC.VALUE_FIELD, ioc.getValue()) + .field(STIX2IOC.SEVERITY_FIELD, ioc.getSeverity()) + .timeField(STIX2IOC.CREATED_FIELD, ioc.getCreated()) + .timeField(STIX2IOC.MODIFIED_FIELD, ioc.getModified()) + .field(STIX2IOC.DESCRIPTION_FIELD, ioc.getDescription()) + .field(STIX2IOC.LABELS_FIELD, ioc.getLabels()) + .field(STIX2IOC.FEED_ID_FIELD, ioc.getFeedId()) + .field(STIX2IOC.FEED_NAME_FIELD, ioc.getFeedName()) + .field(STIX2IOC.SPEC_VERSION_FIELD, ioc.getSpecVersion()) + .field(STIX2IOC.VERSION_FIELD, ioc.getVersion()) + .field(NUM_FINDINGS_FIELD, numFindings) + .endObject(); + } + + public STIX2IOCDto getIoc() { + return ioc; + } + + public void setIoc(STIX2IOCDto ioc) { + this.ioc = ioc; + } + + public long getNumFindings() { + return numFindings; + } + + public void setNumFindings(Long numFindings) { + this.numFindings = numFindings; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/model/DetectorTrigger.java b/src/main/java/org/opensearch/securityanalytics/model/DetectorTrigger.java index 7226cb729..c8881f6c7 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/DetectorTrigger.java +++ b/src/main/java/org/opensearch/securityanalytics/model/DetectorTrigger.java @@ -349,6 +349,90 @@ public Script convertToCondition() { return new Script(condition.toString()); } + public Script convertToConditionForChainedFindings() { + StringBuilder condition = new StringBuilder(); + + boolean triggerFlag = false; + + int size = 0; + if (detectionTypes.contains(RULES_DETECTION_TYPE)) { // trigger should match rules based queries based on conditions + StringBuilder ruleTypeBuilder = new StringBuilder(); + size = ruleTypes.size(); + for (int idx = 0; idx < size; ++idx) { + ruleTypeBuilder.append(String.format(Locale.getDefault(), "query[tag=%s]", ruleTypes.get(idx))); + if (idx < size - 1) { + ruleTypeBuilder.append(" || "); + } + } + if (size > 0) { + condition.append("(").append(ruleTypeBuilder).append(")"); + triggerFlag = true; + } + + StringBuilder ruleNameBuilder = new StringBuilder(); + size = ruleIds.size(); + for (int idx = 0; idx < size; ++idx) { + ruleNameBuilder.append(String.format(Locale.getDefault(), "query[tag=%s]", ruleIds.get(idx))); + if (idx < size - 1) { + ruleNameBuilder.append(" || "); + } + } + if (size > 0) { + if (triggerFlag) { + condition.append(" && ").append("(").append(ruleNameBuilder).append(")"); + } else { + condition.append("(").append(ruleNameBuilder).append(")"); + triggerFlag = true; + } + } + + StringBuilder ruleSevLevelBuilder = new StringBuilder(); + size = ruleSeverityLevels.size(); + for (int idx = 0; idx < size; ++idx) { + ruleSevLevelBuilder.append(String.format(Locale.getDefault(), "query[tag=%s]", ruleSeverityLevels.get(idx))); + if (idx < size - 1) { + ruleSevLevelBuilder.append(" || "); + } + } + + if (size > 0) { + if (triggerFlag) { + condition.append(" && ").append("(").append(ruleSevLevelBuilder).append(")"); + } else { + condition.append("(").append(ruleSevLevelBuilder).append(")"); + triggerFlag = true; + } + } + + StringBuilder tagBuilder = new StringBuilder(); + size = tags.size(); + for (int idx = 0; idx < size; ++idx) { + tagBuilder.append(String.format(Locale.getDefault(), "query[tag=%s]", tags.get(idx))); + if (idx < size - 1) { + ruleSevLevelBuilder.append(" || "); + } + } + + if (size > 0) { + if (triggerFlag) { + condition.append(" && ").append("(").append(tagBuilder).append(")"); + } else { + condition.append("(").append(tagBuilder).append(")"); + } + } + } + if(detectionTypes.contains(THREAT_INTEL_DETECTION_TYPE)) { + StringBuilder threatIntelClauseBuilder = new StringBuilder(); + threatIntelClauseBuilder.append(String.format(Locale.getDefault(), "query[tag=%s]", "threat_intel")); + if (condition.length() > 0) { + condition.append(" || "); + } + condition.append("(").append(threatIntelClauseBuilder).append(")"); + } + + return new Script(condition.toString()); + } + public String getId() { return id; } diff --git a/src/main/java/org/opensearch/securityanalytics/model/LogType.java b/src/main/java/org/opensearch/securityanalytics/model/LogType.java index f70a462e2..1901c7426 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/LogType.java +++ b/src/main/java/org/opensearch/securityanalytics/model/LogType.java @@ -27,6 +27,7 @@ public class LogType implements Writeable { private static final String RAW_FIELD = "raw_field"; public static final String ECS = "ecs"; public static final String OCSF = "ocsf"; + public static final String OCSF11 = "ocsf11"; public static final String IOC_FIELDS = "ioc_fields"; public static final String IOC = "ioc"; public static final String FIELDS = "fields"; @@ -67,7 +68,7 @@ public LogType(Map logTypeAsMap) { if (mappings.size() > 0) { this.mappings = new ArrayList<>(mappings.size()); this.mappings = mappings.stream().map(e -> - new Mapping(e.get(RAW_FIELD), e.get(ECS), e.get(OCSF)) + new Mapping(e.get(RAW_FIELD), e.get(ECS), e.get(OCSF), e.get(OCSF11)) ).collect(Collectors.toList()); } if (logTypeAsMap.containsKey(IOC_FIELDS)) { @@ -120,17 +121,20 @@ public static class Mapping implements Writeable { private String rawField; private String ecs; private String ocsf; + private String ocsf11; public Mapping(StreamInput sin) throws IOException { this.rawField = sin.readString(); this.ecs = sin.readOptionalString(); this.ocsf = sin.readOptionalString(); + this.ocsf11 = sin.readOptionalString(); } - public Mapping(String rawField, String ecs, String ocsf) { + public Mapping(String rawField, String ecs, String ocsf, String ocsf11) { this.rawField = rawField; this.ecs = ecs; this.ocsf = ocsf; + this.ocsf11 = ocsf11; } public String getRawField() { @@ -145,11 +149,14 @@ public String getOcsf() { return ocsf; } + public String getOcsf11() { return ocsf11; } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(rawField); out.writeOptionalString(ecs); out.writeOptionalString(ocsf); + out.writeOptionalString(ocsf11); } public static Mapping readFrom(StreamInput sin) throws IOException { diff --git a/src/main/java/org/opensearch/securityanalytics/model/STIX2IOC.java b/src/main/java/org/opensearch/securityanalytics/model/STIX2IOC.java new file mode 100644 index 000000000..b7f15a094 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/model/STIX2IOC.java @@ -0,0 +1,325 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.model; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.commons.model.STIX2; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import org.opensearch.securityanalytics.util.XContentUtils; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.UUID; + +public class STIX2IOC extends STIX2 implements Writeable, ToXContentObject { + private static final Logger logger = LogManager.getLogger(STIX2IOC.class); + + public static final String NO_ID = ""; + public static final long NO_VERSION = 1L; + + public static final String VERSION_FIELD = "version"; + + private long version = NO_VERSION; + + public STIX2IOC() { + super(); + } + + public STIX2IOC( + String id, + String name, + IOCType type, + String value, + String severity, + Instant created, + Instant modified, + String description, + List labels, + String specVersion, + String feedId, + String feedName, + Long version + ) { + super(StringUtils.isBlank(id) ? UUID.randomUUID().toString() : id, name, type, value, severity, created, modified, description, labels, specVersion, feedId, feedName); + this.version = version; + validate(); + } + + // Constructor used when downloading IOCs from S3 + public STIX2IOC(STIX2 ioc, String feedId, String feedName) { + this( + ioc.getId(), + ioc.getName(), + ioc.getType(), + ioc.getValue(), + ioc.getSeverity(), + ioc.getCreated(), + ioc.getModified(), + ioc.getDescription(), + ioc.getLabels(), + ioc.getSpecVersion(), + feedId, + feedName, + NO_VERSION + ); + } + + public STIX2IOC(StreamInput sin) throws IOException { + this( + sin.readString(), // id + sin.readString(), // name + new IOCType(sin.readString()), // type + sin.readString(), // value + sin.readString(), // severity + sin.readInstant(), // created + sin.readInstant(), // modified + sin.readString(), // description + sin.readStringList(), // labels + sin.readString(), // specVersion + sin.readString(), // feedId + sin.readString(), // feedName + sin.readLong() // version + ); + } + + public STIX2IOC(STIX2IOCDto iocDto) { + this( + iocDto.getId(), + iocDto.getName(), + iocDto.getType(), + iocDto.getValue(), + iocDto.getSeverity(), + iocDto.getCreated(), + iocDto.getModified(), + iocDto.getDescription(), + iocDto.getLabels(), + iocDto.getSpecVersion(), + iocDto.getFeedId(), + iocDto.getFeedName(), + iocDto.getVersion() + ); + } + + public STIX2IOC(STIX2IOCDto ioc, String feedId, String feedName) { + this( + ioc.getId(), + ioc.getName(), + ioc.getType(), + ioc.getValue(), + ioc.getSeverity(), + ioc.getCreated(), + ioc.getModified(), + ioc.getDescription(), + ioc.getLabels(), + ioc.getSpecVersion(), + feedId, + feedName, + NO_VERSION + ); + } + + public static STIX2IOC readFrom(StreamInput sin) throws IOException { + return new STIX2IOC(sin); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(super.getId()); + out.writeString(super.getName()); + out.writeString(super.getType().toString()); + out.writeString(super.getValue()); + out.writeString(super.getSeverity()); + out.writeInstant(super.getCreated()); + out.writeInstant(super.getModified()); + out.writeString(super.getDescription()); + out.writeStringCollection(super.getLabels()); + out.writeString(super.getSpecVersion()); + out.writeString(super.getFeedId()); + out.writeString(super.getFeedName()); + out.writeLong(version); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(ID_FIELD, super.getId()) + .field(NAME_FIELD, super.getName()) + .field(TYPE_FIELD, super.getType().toString()) + .field(VALUE_FIELD, super.getValue()) + .field(SEVERITY_FIELD, super.getSeverity()); + XContentUtils.buildInstantAsField(builder, super.getCreated(), CREATED_FIELD); + XContentUtils.buildInstantAsField(builder, super.getModified(), MODIFIED_FIELD); + return builder.field(DESCRIPTION_FIELD, super.getDescription()) + .field(LABELS_FIELD, super.getLabels()) + .field(SPEC_VERSION_FIELD, super.getSpecVersion()) + .field(FEED_ID_FIELD, super.getFeedId()) + .field(FEED_NAME_FIELD, super.getFeedName()) + .field(VERSION_FIELD, version) + .endObject(); + } + + public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws IOException { + if (id == null) { + id = NO_ID; + } + + if (version == null) { + version = NO_VERSION; + } + + String name = null; + IOCType type = null; + String value = null; + String severity = null; + Instant created = null; + Instant modified = null; + String description = null; + List labels = new ArrayList<>(); + String specVersion = null; + String feedId = null; + String feedName = null; + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case NAME_FIELD: + name = xcp.text(); + break; + case TYPE_FIELD: + String typeString = xcp.text(); + try { + type = new IOCType(typeString); + } catch (Exception e) { + String error = String.format( + "Couldn't parse IOC type '%s' while deserializing STIX2IOC with ID '%s': ", + typeString, + id + ); + logger.error(error, e); + throw new SecurityAnalyticsException(error, RestStatus.BAD_REQUEST, e); + } + break; + case VALUE_FIELD: + value = xcp.text(); + break; + case SEVERITY_FIELD: + severity = xcp.text(); + break; + case CREATED_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + created = null; + } else if (xcp.currentToken().isValue()) { + if (xcp.currentToken() == XContentParser.Token.VALUE_STRING) { + created = Instant.parse(xcp.text()); + } else if (xcp.currentToken() == XContentParser.Token.VALUE_NUMBER) { + created = Instant.ofEpochMilli(xcp.longValue()); + } + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + created = null; + } + break; + case MODIFIED_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + modified = null; + } else if (xcp.currentToken().isValue()) { + if (xcp.currentToken() == XContentParser.Token.VALUE_STRING) { + modified = Instant.parse(xcp.text()); + } else if (xcp.currentToken() == XContentParser.Token.VALUE_NUMBER) { + modified = Instant.ofEpochMilli(xcp.longValue()); + } + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + modified = null; + } + break; + case DESCRIPTION_FIELD: + description = xcp.text(); + break; + case LABELS_FIELD: + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + String entry = xcp.textOrNull(); + if (entry != null) { + labels.add(entry); + } + } + break; + case SPEC_VERSION_FIELD: + specVersion = xcp.text(); + break; + case FEED_ID_FIELD: + feedId = xcp.text(); + break; + case FEED_NAME_FIELD: + feedName = xcp.text(); + break; + default: + xcp.skipChildren(); + } + } + + return new STIX2IOC( + id, + name, + type, + value, + severity, + created, + modified, + description, + labels, + specVersion, + feedId, + feedName, + version + ); + } + + /** + * Validates required fields. + * + * @throws IllegalArgumentException when invalid. + */ + public void validate() throws IllegalArgumentException { + if (super.getType() == null) { + throw new IllegalArgumentException(String.format("[%s] is required.", TYPE_FIELD)); + } else if (!IOCType.supportedType(super.getType().toString())) { + logger.debug("Unsupported IOCType: {}", super.getType().toString()); + throw new IllegalArgumentException(String.format("[%s] is not supported.", TYPE_FIELD)); + } + + if (super.getValue() == null || super.getValue().isEmpty()) { + throw new IllegalArgumentException(String.format("[%s] is required.", VALUE_FIELD)); + } + + if (super.getFeedId() == null || super.getFeedId().isEmpty()) { + throw new IllegalArgumentException(String.format("[%s] is required.", FEED_ID_FIELD)); + } + } + + public Long getVersion() { + return version; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/model/STIX2IOCDto.java b/src/main/java/org/opensearch/securityanalytics/model/STIX2IOCDto.java new file mode 100644 index 000000000..7a59b0ee0 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/model/STIX2IOCDto.java @@ -0,0 +1,360 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.model; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.commons.model.STIX2; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +/** + * A data transfer object for the [STIX2IOC] data model. + */ +public class STIX2IOCDto implements Writeable, ToXContentObject { + private static final Logger logger = LogManager.getLogger(STIX2IOCDto.class); + + private String id; + private String name; + private IOCType type; + private String value; + private String severity; + private Instant created; + private Instant modified; + private String description; + private List labels; + private String specVersion; + private String feedId; + private String feedName; + private long version; + + // No arguments constructor needed for parsing from S3 + public STIX2IOCDto() {} + + public STIX2IOCDto( + String id, + String name, + IOCType type, + String value, + String severity, + Instant created, + Instant modified, + String description, + List labels, + String specVersion, + String feedId, + String feedName, + long version + ) { + this.id = id; + this.name = name; + this.type = type; + this.value = value; + this.severity = severity; + this.created = created; + this.modified = modified; + this.description = description; + this.labels = labels; + this.specVersion = specVersion; + this.feedId = feedId; + this.feedName = feedName; + this.version = version; + } + + public STIX2IOCDto(STIX2IOC ioc) { + this( + ioc.getId(), + ioc.getName(), + ioc.getType(), + ioc.getValue(), + ioc.getSeverity(), + ioc.getCreated(), + ioc.getModified(), + ioc.getDescription(), + ioc.getLabels(), + ioc.getSpecVersion(), + ioc.getFeedId(), + ioc.getFeedName(), + ioc.getVersion() + ); + } + + public STIX2IOCDto(StreamInput sin) throws IOException { + this(new STIX2IOC(sin)); + } + + public static STIX2IOCDto readFrom(StreamInput sin) throws IOException { + return new STIX2IOCDto(sin); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeString(name); + out.writeString(type.toString()); + out.writeString(value); + out.writeString(severity); + out.writeInstant(created); + out.writeInstant(modified); + out.writeString(description); + out.writeStringCollection(labels); + out.writeString(specVersion); + out.writeString(feedId); + out.writeString(feedName); + out.writeLong(version); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field(STIX2IOC.ID_FIELD, id) + .field(STIX2IOC.NAME_FIELD, name) + .field(STIX2IOC.TYPE_FIELD, type.toString()) + .field(STIX2IOC.VALUE_FIELD, value) + .field(STIX2IOC.SEVERITY_FIELD, severity) + .timeField(STIX2IOC.CREATED_FIELD, created) + .timeField(STIX2IOC.MODIFIED_FIELD, modified) + .field(STIX2IOC.DESCRIPTION_FIELD, description) + .field(STIX2IOC.LABELS_FIELD, labels) + .field(STIX2IOC.SPEC_VERSION_FIELD, specVersion) + .field(STIX2IOC.FEED_ID_FIELD, feedId) + .field(STIX2IOC.FEED_NAME_FIELD, feedName) + .field(STIX2IOC.VERSION_FIELD, version) + .endObject(); + } + + public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) throws IOException { + if (id == null) { + id = STIX2IOC.NO_ID; + } + + if (version == null) { + version = STIX2IOC.NO_VERSION; + } + + String name = null; + IOCType type = null; + String value = null; + String severity = null; + Instant created = null; + Instant modified = null; + String description = null; + List labels = new ArrayList<>(); + String specVersion = null; + String feedId = null; + String feedName = null; + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case STIX2.ID_FIELD: + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { + id = xcp.text(); + } + break; + case STIX2IOC.VERSION_FIELD: + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { + version = xcp.longValue(); + } + break; + case STIX2.NAME_FIELD: + name = xcp.text(); + break; + case STIX2.TYPE_FIELD: + String typeString = xcp.text(); + try { + type = new IOCType(typeString); + } catch (Exception e) { + String error = String.format( + "Couldn't parse IOC type '%s' while deserializing STIX2IOCDto with ID '%s': ", + typeString, + id + ); + logger.error(error, e); + throw new SecurityAnalyticsException(error, RestStatus.BAD_REQUEST, e); + } + break; + case STIX2.VALUE_FIELD: + value = xcp.text(); + break; + case STIX2.SEVERITY_FIELD: + severity = xcp.text(); + break; + case STIX2.CREATED_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + created = null; + } else if (xcp.currentToken().isValue()) { + if (xcp.currentToken() == XContentParser.Token.VALUE_STRING) { + created = Instant.parse(xcp.text()); + } else if (xcp.currentToken() == XContentParser.Token.VALUE_NUMBER) { + created = Instant.ofEpochMilli(xcp.longValue()); + } + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + created = null; + } + break; + case STIX2.MODIFIED_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + modified = null; + } else if (xcp.currentToken().isValue()) { + if (xcp.currentToken() == XContentParser.Token.VALUE_STRING) { + modified = Instant.parse(xcp.text()); + } else if (xcp.currentToken() == XContentParser.Token.VALUE_NUMBER) { + modified = Instant.ofEpochMilli(xcp.longValue()); + } + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + modified = null; + } + break; + case STIX2.DESCRIPTION_FIELD: + description = xcp.text(); + break; + case STIX2.LABELS_FIELD: + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + String entry = xcp.textOrNull(); + if (entry != null) { + labels.add(entry); + } + } + break; + case STIX2.SPEC_VERSION_FIELD: + specVersion = xcp.text(); + break; + case STIX2IOC.FEED_ID_FIELD: + feedId = xcp.text(); + break; + case STIX2IOC.FEED_NAME_FIELD: + feedName = xcp.text(); + break; + default: + xcp.skipChildren(); + } + } + + return new STIX2IOCDto( + id, + name, + type, + value, + severity, + created, + modified, + description, + labels, + specVersion, + feedId, + feedName, + version + ); + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public IOCType getType() { + return type; + } + + public void setType(IOCType type) { + this.type = type; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public String getSeverity() { + return severity; + } + + public void setSeverity(String severity) { + this.severity = severity; + } + + public Instant getCreated() { + return created; + } + + public void setCreated(Instant created) { + this.created = created; + } + + public Instant getModified() { + return modified; + } + + public void setModified(Instant modified) { + this.modified = modified; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public List getLabels() { + return labels; + } + + public String getSpecVersion() { + return specVersion; + } + + public String getFeedId() { + return feedId; + } + + public String getFeedName() { + return feedName; + } + + public long getVersion() { + return version; + } + + public void setVersion(Long version) { + this.version = version; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/model/threatintel/BaseEntity.java b/src/main/java/org/opensearch/securityanalytics/model/threatintel/BaseEntity.java new file mode 100644 index 000000000..e72fac958 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/model/threatintel/BaseEntity.java @@ -0,0 +1,18 @@ +package org.opensearch.securityanalytics.model.threatintel; + +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; + +public abstract class BaseEntity implements Writeable, ToXContentObject { + @Override + public abstract void writeTo(StreamOutput out) throws IOException; + + @Override + public abstract XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException; + + public abstract String getId(); +} diff --git a/src/main/java/org/opensearch/securityanalytics/model/threatintel/IocFinding.java b/src/main/java/org/opensearch/securityanalytics/model/threatintel/IocFinding.java new file mode 100644 index 000000000..a6ffb5bb3 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/model/threatintel/IocFinding.java @@ -0,0 +1,250 @@ +package org.opensearch.securityanalytics.model.threatintel; + +import org.apache.commons.lang3.StringUtils; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; + +/** + * IoC Match provides mapping of the IoC Value to the list of docs that contain the ioc in a given execution of IoC_Scan_job + * It's the inverse of an IoC finding which maps a document to list of IoC's + */ +public class IocFinding extends BaseEntity { + //TODO implement IoC_Match interface from security-analytics-commons + public static final String ID_FIELD = "id"; + public static final String RELATED_DOC_IDS_FIELD = "related_doc_ids"; + public static final String IOC_WITH_FEED_IDS_FIELD = "ioc_feed_ids"; + public static final String MONITOR_ID_FIELD = "monitor_id"; + public static final String MONITOR_NAME_FIELD = "monitor_name"; + public static final String IOC_VALUE_FIELD = "ioc_value"; + public static final String IOC_TYPE_FIELD = "ioc_type"; + public static final String TIMESTAMP_FIELD = "timestamp"; + public static final String EXECUTION_ID_FIELD = "execution_id"; + + private final String id; + private final List relatedDocIds; + private final List iocWithFeeds; + private final String monitorId; + private final String monitorName; + private final String iocValue; + private final String iocType; + private final Instant timestamp; + private final String executionId; + + public IocFinding(String id, List relatedDocIds, List iocWithFeeds, String monitorId, + String monitorName, String iocValue, String iocType, Instant timestamp, String executionId) { + validateIoCMatch(id, monitorId, monitorName, iocValue, timestamp, executionId, relatedDocIds); + this.id = id; + this.relatedDocIds = relatedDocIds; + this.iocWithFeeds = iocWithFeeds; + this.monitorId = monitorId; + this.monitorName = monitorName; + this.iocValue = iocValue; + this.iocType = iocType; + this.timestamp = timestamp; + this.executionId = executionId; + } + + public IocFinding(StreamInput in) throws IOException { + id = in.readString(); + relatedDocIds = in.readStringList(); + iocWithFeeds = in.readList(IocWithFeeds::readFrom); + monitorId = in.readString(); + monitorName = in.readString(); + iocValue = in.readString(); + iocType = in.readString(); + timestamp = in.readInstant(); + executionId = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeStringCollection(relatedDocIds); + out.writeCollection(iocWithFeeds); + out.writeString(monitorId); + out.writeString(monitorName); + out.writeString(iocValue); + out.writeString(iocType); + out.writeInstant(timestamp); + out.writeOptionalString(executionId); + } + + public Map asTemplateArg() { + return Map.of( + ID_FIELD, id, + RELATED_DOC_IDS_FIELD, relatedDocIds, + IOC_WITH_FEED_IDS_FIELD, iocWithFeeds.stream().map(IocWithFeeds::asTemplateArg).collect(Collectors.toList()), + MONITOR_ID_FIELD, monitorId, + MONITOR_NAME_FIELD, monitorName, + IOC_VALUE_FIELD, iocValue, + IOC_TYPE_FIELD, iocType, + TIMESTAMP_FIELD, timestamp, + EXECUTION_ID_FIELD, executionId + ); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(ID_FIELD, id) + .field(RELATED_DOC_IDS_FIELD, relatedDocIds) + .field(IOC_WITH_FEED_IDS_FIELD, iocWithFeeds) + .field(MONITOR_ID_FIELD, monitorId) + .field(MONITOR_NAME_FIELD, monitorName) + .field(IOC_VALUE_FIELD, iocValue) + .field(IOC_TYPE_FIELD, iocType) + .field(TIMESTAMP_FIELD, timestamp.toEpochMilli()) + .field(EXECUTION_ID_FIELD, executionId) + .endObject(); + return builder; + } + + public String getId() { + return id; + } + + public List getRelatedDocIds() { + return relatedDocIds; + } + + public List getFeedIds() { + return iocWithFeeds; + } + + public String getMonitorId() { + return monitorId; + } + + public String getMonitorName() { + return monitorName; + } + + public String getIocValue() { + return iocValue; + } + + public String getIocType() { + return iocType; + } + + public Instant getTimestamp() { + return timestamp; + } + + public String getExecutionId() { + return executionId; + } + + public static IocFinding parse(XContentParser xcp) throws IOException { + String id = null; + List relatedDocIds = new ArrayList<>(); + List feedIds = new ArrayList<>(); + String monitorId = null; + String monitorName = null; + String iocValue = null; + String iocType = null; + Instant timestamp = null; + String executionId = null; + + ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case ID_FIELD: + id = xcp.text(); + break; + case RELATED_DOC_IDS_FIELD: + ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + relatedDocIds.add(xcp.text()); + } + break; + case IOC_WITH_FEED_IDS_FIELD: + ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + feedIds.add(IocWithFeeds.parse(xcp)); + } + break; + case MONITOR_ID_FIELD: + monitorId = xcp.textOrNull(); + break; + case MONITOR_NAME_FIELD: + monitorName = xcp.textOrNull(); + break; + case IOC_VALUE_FIELD: + iocValue = xcp.textOrNull(); + break; + case IOC_TYPE_FIELD: + iocType = xcp.textOrNull(); + break; + case TIMESTAMP_FIELD: + try { + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + timestamp = null; + } else if (xcp.currentToken().isValue()) { + timestamp = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + timestamp = null; + } + break; + } catch (Exception e) { + throw new IllegalArgumentException("failed to parse timestamp in IoC Match object"); + } + case EXECUTION_ID_FIELD: + executionId = xcp.textOrNull(); + break; + default: + xcp.skipChildren(); + } + } + + return new IocFinding(id, relatedDocIds, feedIds, monitorId, monitorName, iocValue, iocType, timestamp, executionId); + } + + public static IocFinding readFrom(StreamInput in) throws IOException { + return new IocFinding(in); + } + + + private static void validateIoCMatch(String id, String iocScanJobId, String iocScanName, String iocValue, Instant timestamp, String executionId, List relatedDocIds) { + if (StringUtils.isBlank(id)) { + throw new IllegalArgumentException("id cannot be empty in IoC_Match Object"); + } + if (StringUtils.isBlank(iocValue)) { + throw new IllegalArgumentException("ioc_value cannot be empty in IoC_Match Object"); + } + if (StringUtils.isBlank(iocValue)) { + throw new IllegalArgumentException("ioc_value cannot be empty in IoC_Match Object"); + } + if (StringUtils.isBlank(iocScanJobId)) { + throw new IllegalArgumentException("monitor_id cannot be empty in IoC_Match Object"); + } + if (StringUtils.isBlank(iocScanName)) { + throw new IllegalArgumentException("monitor_name cannot be empty in IoC_Match Object"); + } + if (StringUtils.isBlank(executionId)) { + throw new IllegalArgumentException("execution_id cannot be empty in IoC_Match Object"); + } + if (timestamp == null) { + throw new IllegalArgumentException("timestamp cannot be null in IoC_Match Object"); + } + if (relatedDocIds == null || relatedDocIds.isEmpty()) { + throw new IllegalArgumentException("related_doc_ids cannot be null or empty in IoC_Match Object"); + } + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/model/threatintel/IocWithFeeds.java b/src/main/java/org/opensearch/securityanalytics/model/threatintel/IocWithFeeds.java new file mode 100644 index 000000000..055ec0e55 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/model/threatintel/IocWithFeeds.java @@ -0,0 +1,155 @@ +package org.opensearch.securityanalytics.model.threatintel; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; + +/** + * container class to store a tuple of feed id, ioc id and index. + */ +public class IocWithFeeds implements Writeable, ToXContent { + + private static final String FEED_ID_FIELD = "feed_id"; + + private static final String FEED_NAME_FIELD = "feed_name"; + + private static final String IOC_ID_FIELD = "ioc_id"; + + private static final String INDEX_FIELD = "index"; + + private final String feedId; + private final String feedName; + + private final String iocId; + + private final String index; + + public IocWithFeeds(String iocId, String feedId, String feedName, String index) { + this.iocId = iocId; + this.feedId = feedId; + this.feedName = feedName; + this.index = index; + } + + public IocWithFeeds(StreamInput sin) throws IOException { + this.iocId = sin.readString(); + this.feedId = sin.readString(); + this.feedName = sin.readString(); + this.index = sin.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(iocId); + out.writeString(feedId); + out.writeString(feedName); + out.writeString(index); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(IOC_ID_FIELD, iocId) + .field(FEED_ID_FIELD, feedId) + .field(FEED_NAME_FIELD, feedName) + .field(INDEX_FIELD, index) + .endObject(); + return builder; + } + + public Map asTemplateArg() { + return Map.of( + FEED_ID_FIELD, feedId, + FEED_NAME_FIELD, feedId, + IOC_ID_FIELD, iocId, + INDEX_FIELD, index + ); + } + + public String getIocId() { + return iocId; + } + + public String getFeedId() { + return feedId; + } + + public String getFeedName() { + return feedName; + } + + public String getIndex() { + return index; + } + + public static IocWithFeeds parse(XContentParser xcp) throws IOException { + String iocId = null; + String feedId = null; + String feedName = null; + String index = null; + + ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case IOC_ID_FIELD: + iocId = xcp.text(); + break; + case FEED_ID_FIELD: + feedId = xcp.text(); + break; + case FEED_NAME_FIELD: + feedName = xcp.text(); + break; + case INDEX_FIELD: + index = xcp.text(); + break; + default: + xcp.skipChildren(); + } + } + return new IocWithFeeds(iocId, feedId, feedName, index); + } + + public static IocWithFeeds readFrom(StreamInput sin) throws IOException { + return new IocWithFeeds(sin); + } + + @Override + public int hashCode() { + return Objects.hash(feedId, feedName, index, iocId); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + IocWithFeeds that = (IocWithFeeds) o; + + if (feedId != null ? !feedId.equals(that.feedId) : that.feedId != null) return false; + if (iocId != null ? !iocId.equals(that.iocId) : that.iocId != null) return false; + return index != null ? index.equals(that.index) : that.index == null; + } + + @Override + public String toString() { + return "IocWithFeeds{" + + "feedId='" + feedId + '\'' + + "feedName='" + feedName + '\'' + + ", iocId='" + iocId + '\'' + + ", index='" + index + '\'' + + '}'; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/model/threatintel/ThreatIntelAlert.java b/src/main/java/org/opensearch/securityanalytics/model/threatintel/ThreatIntelAlert.java new file mode 100644 index 000000000..fa1f2ddcb --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/model/threatintel/ThreatIntelAlert.java @@ -0,0 +1,655 @@ +package org.opensearch.securityanalytics.model.threatintel; + +import org.opensearch.commons.alerting.model.ActionExecutionResult; +import org.opensearch.commons.alerting.model.Alert; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.seqno.SequenceNumbers; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelAlertDto; +import org.opensearch.securityanalytics.util.XContentUtils; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.securityanalytics.util.XContentUtils.getInstant; + +public class ThreatIntelAlert extends BaseEntity { + + public static final String ALERT_ID_FIELD = "id"; + public static final String SCHEMA_VERSION_FIELD = "schema_version"; + public static final String ALERT_VERSION_FIELD = "version"; + public static final String USER_FIELD = "user"; + public static final String TRIGGER_NAME_FIELD = "trigger_name"; + public static final String TRIGGER_ID_FIELD = "trigger_id"; + public static final String MONITOR_ID_FIELD = "monitor_id"; + public static final String MONITOR_NAME_FIELD = "monitor_name"; + public static final String STATE_FIELD = "state"; + public static final String START_TIME_FIELD = "start_time"; + public static final String END_TIME_FIELD = "end_time"; + public static final String LAST_UPDATED_TIME_FIELD = "last_updated_time"; + public static final String ACKNOWLEDGED_TIME_FIELD = "acknowledged_time"; + public static final String ERROR_MESSAGE_FIELD = "error_message"; + public static final String SEVERITY_FIELD = "severity"; + public static final String ACTION_EXECUTION_RESULTS_FIELD = "action_execution_results"; + public static final String IOC_VALUE_FIELD = "ioc_value"; + public static final String IOC_TYPE_FIELD = "ioc_type"; + public static final String FINDING_IDS_FIELD = "finding_ids"; + public static final String SEQ_NO_FIELD = "seq_no"; + public static final String PRIMARY_TERM_FIELD = "primary_term"; + public static final String NO_ID = ""; + public static final long NO_VERSION = 1L; + public static final long NO_SCHEMA_VERSION = 0; + + private final String id; + private final long version; + private final long schemaVersion; + private final long seqNo; + private final long primaryTerm; + private final User user; + private final String triggerName; + private final String triggerId; + private final String monitorId; + private final String monitorName; + private final Alert.State state; + private final Instant startTime; + private final Instant endTime; + private final Instant acknowledgedTime; + private final Instant lastUpdatedTime; + private final String errorMessage; + private final String severity; + private final String iocValue; + private final String iocType; + private final List actionExecutionResults; + private List findingIds; + + public ThreatIntelAlert( + String id, + long version, + long schemaVersion, + User user, + String triggerId, + String triggerName, + String monitorId, + String monitorName, + Alert.State state, + Instant startTime, + Instant endTime, + Instant lastUpdatedTime, + Instant acknowledgedTime, + String errorMessage, + String severity, + String iocValue, + String iocType, + List actionExecutionResults, + List findingIds + ) { + + this.id = id != null ? id : NO_ID; + this.version = version != 0 ? version : NO_VERSION; + this.schemaVersion = schemaVersion; + this.seqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; + this.primaryTerm = SequenceNumbers.UNASSIGNED_PRIMARY_TERM; + this.user = user; + this.triggerId = triggerId; + this.triggerName = triggerName; + this.monitorId = monitorId; + this.monitorName = monitorName; + this.state = state; + this.startTime = startTime; + this.endTime = endTime; + this.acknowledgedTime = acknowledgedTime; + this.errorMessage = errorMessage; + this.severity = severity; + this.iocValue = iocValue; + this.iocType = iocType; + this.actionExecutionResults = actionExecutionResults; + this.lastUpdatedTime = lastUpdatedTime; + this.findingIds = findingIds; + } + + public ThreatIntelAlert( + String id, + long version, + long schemaVersion, + long seqNo, + long primaryTerm, + User user, + String triggerId, + String triggerName, + String monitorId, + String monitorName, + Alert.State state, + Instant startTime, + Instant endTime, + Instant lastUpdatedTime, + Instant acknowledgedTime, + String errorMessage, + String severity, + String iocValue, + String iocType, + List actionExecutionResults, + List findingIds + ) { + + this.id = id != null ? id : NO_ID; + this.version = version != 0 ? version : NO_VERSION; + this.schemaVersion = schemaVersion; + this.seqNo = seqNo; + this.primaryTerm = primaryTerm; + this.user = user; + this.triggerId = triggerId; + this.triggerName = triggerName; + this.monitorId = monitorId; + this.monitorName = monitorName; + this.state = state; + this.startTime = startTime; + this.endTime = endTime; + this.acknowledgedTime = acknowledgedTime; + this.errorMessage = errorMessage; + this.severity = severity; + this.iocValue = iocValue; + this.iocType = iocType; + this.actionExecutionResults = actionExecutionResults; + this.lastUpdatedTime = lastUpdatedTime; + this.findingIds = findingIds; + } + + public ThreatIntelAlert(StreamInput sin) throws IOException { + this.id = sin.readString(); + this.version = sin.readLong(); + this.schemaVersion = sin.readLong(); + this.seqNo = sin.readLong(); + this.primaryTerm = sin.readLong(); + this.user = sin.readBoolean() ? new User(sin) : null; + this.triggerId = sin.readString(); + this.triggerName = sin.readString(); + this.monitorId = sin.readString(); + this.monitorName = sin.readString(); + this.state = sin.readEnum(Alert.State.class); + this.startTime = sin.readInstant(); + this.endTime = sin.readOptionalInstant(); + this.acknowledgedTime = sin.readOptionalInstant(); + this.errorMessage = sin.readOptionalString(); + this.severity = sin.readString(); + this.actionExecutionResults = sin.readList(ActionExecutionResult::new); + this.lastUpdatedTime = sin.readOptionalInstant(); + this.iocType = sin.readString(); + this.iocValue = sin.readString(); + this.findingIds = sin.readStringList(); + } + + public ThreatIntelAlert(ThreatIntelAlert currentAlert, List findingIds) { + this.findingIds = findingIds; + this.id = currentAlert.id; + this.version = currentAlert.version; + this.schemaVersion = currentAlert.schemaVersion; + this.seqNo =currentAlert.seqNo; + this.primaryTerm =currentAlert.primaryTerm; + this.user = currentAlert.user; + this.triggerId = currentAlert.triggerId; + this.triggerName = currentAlert.triggerName; + this.monitorId = currentAlert.monitorId; + this.monitorName = currentAlert.monitorName; + this.state = currentAlert.state; + this.startTime = currentAlert.startTime; + this.endTime = currentAlert.endTime; + this.acknowledgedTime = currentAlert.acknowledgedTime; + this.errorMessage = currentAlert.errorMessage; + this.severity = currentAlert.severity; + this.iocValue = currentAlert.iocValue; + this.iocType = currentAlert.iocType; + this.actionExecutionResults = currentAlert.actionExecutionResults; + this.lastUpdatedTime = Instant.now(); + } + + public static ThreatIntelAlert updateStatus(ThreatIntelAlert currentAlert, Alert.State newState) { + return new ThreatIntelAlert( + currentAlert.id, + currentAlert.version, + currentAlert.schemaVersion, + currentAlert.seqNo, + currentAlert.primaryTerm, + currentAlert.user, + currentAlert.triggerId, + currentAlert.triggerName, + currentAlert.monitorId, + currentAlert.monitorName, + newState, + currentAlert.startTime, + newState.equals(Alert.State.COMPLETED) ? Instant.now() : currentAlert.endTime, + Instant.now(), + newState.equals(Alert.State.ACKNOWLEDGED) ? Instant.now() : currentAlert.endTime, + currentAlert.errorMessage, + currentAlert.severity, + currentAlert.iocValue, + currentAlert.iocType, + currentAlert.actionExecutionResults, + currentAlert.getFindingIds() + ); + } + + public boolean isAcknowledged() { + return state == Alert.State.ACKNOWLEDGED; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(version); + out.writeLong(schemaVersion); + out.writeLong(seqNo); + out.writeLong(primaryTerm); + out.writeBoolean(user != null); + if (user != null) { + user.writeTo(out); + } + out.writeString(triggerId); + out.writeString(triggerName); + out.writeString(monitorId); + out.writeString(monitorName); + out.writeEnum(state); + out.writeInstant(startTime); + out.writeOptionalInstant(endTime); + out.writeOptionalInstant(acknowledgedTime); + out.writeOptionalString(errorMessage); + out.writeString(severity); + out.writeCollection(actionExecutionResults); + out.writeOptionalInstant(lastUpdatedTime); + out.writeString(iocType); + out.writeString(iocValue); + out.writeStringCollection(findingIds); + } + + public static ThreatIntelAlert parse(XContentParser xcp, long version, long seqNo, long primaryTerm) throws IOException { + String id = NO_ID; + long schemaVersion = NO_SCHEMA_VERSION; + User user = null; + String triggerId = null; + String triggerName = null; + String monitorId = null; + String monitorName = null; + Alert.State state = null; + Instant startTime = null; + String severity = null; + Instant endTime = null; + Instant acknowledgedTime = null; + Instant lastUpdatedTime = null; + String errorMessage = null; + List actionExecutionResults = new ArrayList<>(); + String iocValue = null; + String iocType = null; + List findingIds = new ArrayList<>(); + + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + switch (fieldName) { + case USER_FIELD: + user = xcp.currentToken() == XContentParser.Token.VALUE_NULL ? null : User.parse(xcp); + break; + case ALERT_ID_FIELD: + id = xcp.text(); + break; + case IOC_VALUE_FIELD: + iocValue = xcp.textOrNull(); + break; + case IOC_TYPE_FIELD: + iocType = xcp.textOrNull(); + break; + case ALERT_VERSION_FIELD: + version = xcp.longValue(); + break; + case SCHEMA_VERSION_FIELD: + schemaVersion = xcp.intValue(); + break; + case SEQ_NO_FIELD: + seqNo = xcp.longValue(); + break; + case PRIMARY_TERM_FIELD: + primaryTerm = xcp.longValue(); + break; + case TRIGGER_ID_FIELD: + triggerId = xcp.text(); + break; + case TRIGGER_NAME_FIELD: + triggerName = xcp.text(); + break; + case MONITOR_ID_FIELD: + monitorId = xcp.text(); + break; + case MONITOR_NAME_FIELD: + monitorName = xcp.text(); + break; + case STATE_FIELD: + state = Alert.State.valueOf(xcp.text()); + break; + case ERROR_MESSAGE_FIELD: + errorMessage = xcp.textOrNull(); + break; + case SEVERITY_FIELD: + severity = xcp.text(); + break; + case ACTION_EXECUTION_RESULTS_FIELD: + ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + actionExecutionResults.add(ActionExecutionResult.parse(xcp)); + } + break; + case START_TIME_FIELD: + startTime = getInstant(xcp); + break; + case END_TIME_FIELD: + endTime = getInstant(xcp); + break; + case ACKNOWLEDGED_TIME_FIELD: + acknowledgedTime = getInstant(xcp); + break; + case LAST_UPDATED_TIME_FIELD: + lastUpdatedTime = getInstant(xcp); + break; + case FINDING_IDS_FIELD: + ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + findingIds.add(xcp.text()); + } + default: + xcp.skipChildren(); + } + } + + return new ThreatIntelAlert(id, + version, + schemaVersion, + seqNo, + primaryTerm, + user, + triggerId, + triggerName, + monitorId, + monitorName, + state, + startTime, + endTime, + acknowledgedTime, + lastUpdatedTime, + errorMessage, + severity, + iocValue, iocType, actionExecutionResults, findingIds); + } + + public static ThreatIntelAlert parse(XContentParser xcp, long version) throws IOException { + String id = NO_ID; + long schemaVersion = NO_SCHEMA_VERSION; + long seqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; + long primaryTerm = SequenceNumbers.UNASSIGNED_PRIMARY_TERM; + User user = null; + String triggerId = null; + String triggerName = null; + String monitorId = null; + String monitorName = null; + Alert.State state = null; + Instant startTime = null; + String severity = null; + Instant endTime = null; + Instant acknowledgedTime = null; + Instant lastUpdatedTime = null; + String errorMessage = null; + List actionExecutionResults = new ArrayList<>(); + String iocValue = null; + String iocType = null; + List findingIds = new ArrayList<>(); + + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + switch (fieldName) { + case USER_FIELD: + user = xcp.currentToken() == XContentParser.Token.VALUE_NULL ? null : User.parse(xcp); + break; + case ALERT_ID_FIELD: + id = xcp.text(); + break; + case IOC_VALUE_FIELD: + iocValue = xcp.textOrNull(); + break; + case IOC_TYPE_FIELD: + iocType = xcp.textOrNull(); + break; + case ALERT_VERSION_FIELD: + version = xcp.longValue(); + break; + case SCHEMA_VERSION_FIELD: + schemaVersion = xcp.intValue(); + break; + case SEQ_NO_FIELD: + seqNo = xcp.longValue(); + break; + case PRIMARY_TERM_FIELD: + primaryTerm = xcp.longValue(); + break; + case TRIGGER_ID_FIELD: + triggerId = xcp.text(); + break; + case TRIGGER_NAME_FIELD: + triggerName = xcp.text(); + break; + case MONITOR_ID_FIELD: + monitorId = xcp.text(); + break; + case MONITOR_NAME_FIELD: + monitorName = xcp.text(); + break; + case STATE_FIELD: + state = Alert.State.valueOf(xcp.text()); + break; + case ERROR_MESSAGE_FIELD: + errorMessage = xcp.textOrNull(); + break; + case SEVERITY_FIELD: + severity = xcp.text(); + break; + case ACTION_EXECUTION_RESULTS_FIELD: + ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + actionExecutionResults.add(ActionExecutionResult.parse(xcp)); + } + break; + case START_TIME_FIELD: + startTime = getInstant(xcp); + break; + case END_TIME_FIELD: + endTime = getInstant(xcp); + break; + case ACKNOWLEDGED_TIME_FIELD: + acknowledgedTime = getInstant(xcp); + break; + case LAST_UPDATED_TIME_FIELD: + lastUpdatedTime = getInstant(xcp); + break; + case FINDING_IDS_FIELD: + ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + findingIds.add(xcp.text()); + } + default: + xcp.skipChildren(); + } + } + + return new ThreatIntelAlert(id, + version, + schemaVersion, + seqNo, + primaryTerm, + user, + triggerId, + triggerName, + monitorId, + monitorName, + state, + startTime, + endTime, + acknowledgedTime, + lastUpdatedTime, + errorMessage, + severity, + iocValue, iocType, actionExecutionResults, findingIds); + } + + public static Alert readFrom(StreamInput sin) throws IOException { + return new Alert(sin); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return createXContentBuilder(builder, true); + } + + @Override + public String getId() { + return id; + } + + public XContentBuilder toXContentWithUser(XContentBuilder builder) throws IOException { + return createXContentBuilder(builder, false); + } + + private XContentBuilder createXContentBuilder(XContentBuilder builder, boolean secure) throws IOException { + builder.startObject() + .field(ALERT_ID_FIELD, id) + .field(ALERT_VERSION_FIELD, version) + .field(SCHEMA_VERSION_FIELD, schemaVersion) + .field(SEQ_NO_FIELD, seqNo) + .field(PRIMARY_TERM_FIELD, primaryTerm) + .field(TRIGGER_NAME_FIELD, triggerName) + .field(TRIGGER_ID_FIELD, triggerId) + .field(MONITOR_ID_FIELD, monitorId) + .field(MONITOR_NAME_FIELD, monitorName) + .field(STATE_FIELD, state) + .field(ERROR_MESSAGE_FIELD, errorMessage) + .field(IOC_VALUE_FIELD, iocValue) + .field(IOC_TYPE_FIELD, iocType) + .field(SEVERITY_FIELD, severity) + .field(ACTION_EXECUTION_RESULTS_FIELD, actionExecutionResults.toArray()) + .field(FINDING_IDS_FIELD, findingIds.toArray(new String[0])); + XContentUtils.buildInstantAsField(builder, acknowledgedTime, ACKNOWLEDGED_TIME_FIELD); + XContentUtils.buildInstantAsField(builder, lastUpdatedTime, LAST_UPDATED_TIME_FIELD); + XContentUtils.buildInstantAsField(builder, startTime, START_TIME_FIELD); + XContentUtils.buildInstantAsField(builder, endTime, END_TIME_FIELD); + if (!secure) { + if (user == null) { + builder.nullField(USER_FIELD); + } else { + builder.field(USER_FIELD, user); + } + } + return builder.endObject(); + } + + public Map asTemplateArg() { + Map map = new HashMap<>(); + map.put(ACKNOWLEDGED_TIME_FIELD, acknowledgedTime != null ? acknowledgedTime.toEpochMilli() : null); + map.put(ALERT_ID_FIELD, id); + map.put(ALERT_VERSION_FIELD, version); + map.put(END_TIME_FIELD, endTime != null ? endTime.toEpochMilli() : null); + map.put(ERROR_MESSAGE_FIELD, errorMessage); + map.put(SEVERITY_FIELD, severity); + map.put(START_TIME_FIELD, startTime.toEpochMilli()); + map.put(STATE_FIELD, state.toString()); + map.put(TRIGGER_ID_FIELD, triggerId); + map.put(TRIGGER_NAME_FIELD, triggerName); + map.put(FINDING_IDS_FIELD, findingIds); + map.put(LAST_UPDATED_TIME_FIELD, lastUpdatedTime); + map.put(IOC_TYPE_FIELD, iocType); + map.put(IOC_VALUE_FIELD, iocValue); + return map; + } + + public long getVersion() { + return version; + } + + public long getSchemaVersion() { + return schemaVersion; + } + + public User getUser() { + return user; + } + + public String getTriggerName() { + return triggerName; + } + + public Alert.State getState() { + return state; + } + + public Instant getStartTime() { + return startTime; + } + + public Instant getEndTime() { + return endTime; + } + + public Instant getAcknowledgedTime() { + return acknowledgedTime; + } + + public String getErrorMessage() { + return errorMessage; + } + + public String getSeverity() { + return severity; + } + + public List getActionExecutionResults() { + return actionExecutionResults; + } + + public String getTriggerId() { + return triggerId; + } + + public Instant getLastUpdatedTime() { + return lastUpdatedTime; + } + + public String getIocValue() { + return iocValue; + } + + public String getIocType() { + return iocType; + } + + public List getFindingIds() { + return findingIds; + } + + public String getMonitorId() { + return monitorId; + } + + public String getMonitorName() { + return monitorName; + } + + public long getSeqNo() { + return seqNo; + } + + public long getPrimaryTerm() { + return primaryTerm; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/resthandler/RestAcknowledgeAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/resthandler/RestAcknowledgeAlertsAction.java index 2a49e49cb..ed226190a 100644 --- a/src/main/java/org/opensearch/securityanalytics/resthandler/RestAcknowledgeAlertsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/resthandler/RestAcknowledgeAlertsAction.java @@ -25,7 +25,7 @@ /** * Acknowledge list of alerts generated by a detector. */ -public class RestAcknowledgeAlertsAction extends BaseRestHandler { +public class RestAcknowledgeAlertsAction extends BaseRestHandler { @Override public String getName() { return "ack_detector_alerts_action"; diff --git a/src/main/java/org/opensearch/securityanalytics/resthandler/RestGetAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/resthandler/RestGetAlertsAction.java index 19322d0cd..0276db801 100644 --- a/src/main/java/org/opensearch/securityanalytics/resthandler/RestGetAlertsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/resthandler/RestGetAlertsAction.java @@ -5,6 +5,8 @@ package org.opensearch.securityanalytics.resthandler; import java.io.IOException; +import java.time.DateTimeException; +import java.time.Instant; import java.util.List; import java.util.Locale; import org.opensearch.client.node.NodeClient; @@ -45,6 +47,26 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli int startIndex = request.paramAsInt("startIndex", 0); String searchString = request.param("searchString", ""); + Instant startTime = null; + String startTimeParam = request.param("startTime"); + if (startTimeParam != null && !startTimeParam.isEmpty()) { + try { + startTime = Instant.ofEpochMilli(Long.parseLong(startTimeParam)); + } catch (NumberFormatException | NullPointerException | DateTimeException e) { + startTime = Instant.now(); + } + } + + Instant endTime = null; + String endTimeParam = request.param("endTime"); + if (endTimeParam != null && !endTimeParam.isEmpty()) { + try { + endTime = Instant.ofEpochMilli(Long.parseLong(endTimeParam)); + } catch (NumberFormatException | NullPointerException | DateTimeException e) { + endTime = Instant.now(); + } + } + Table table = new Table( sortOrder, sortString, @@ -59,7 +81,9 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli detectorType, table, severityLevel, - alertState + alertState, + startTime, + endTime ); return channel -> client.execute( diff --git a/src/main/java/org/opensearch/securityanalytics/resthandler/RestIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/resthandler/RestIndexDetectorAction.java index 6fac7a078..f0f8d7fc0 100644 --- a/src/main/java/org/opensearch/securityanalytics/resthandler/RestIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/resthandler/RestIndexDetectorAction.java @@ -75,13 +75,13 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli } private static void validateDetectorTriggers(Detector detector) { - if(detector.getTriggers() != null) { + if (detector.getTriggers() != null) { for (DetectorTrigger trigger : detector.getTriggers()) { - if(trigger.getDetectionTypes().isEmpty()) - throw new IllegalArgumentException(String.format(Locale.ROOT,"Trigger [%s] should mention at least one detection type but found none", trigger.getName())); + if (trigger.getDetectionTypes().isEmpty()) + throw new IllegalArgumentException(String.format(Locale.ROOT, "Trigger [%s] should mention at least one detection type but found none", trigger.getName())); for (String detectionType : trigger.getDetectionTypes()) { - if(false == (DetectorTrigger.THREAT_INTEL_DETECTION_TYPE.equals(detectionType) || DetectorTrigger.RULES_DETECTION_TYPE.equals(detectionType))) { - throw new IllegalArgumentException(String.format(Locale.ROOT,"Trigger [%s] has unsupported detection type [%s]", trigger.getName(), detectionType)); + if (false == (DetectorTrigger.THREAT_INTEL_DETECTION_TYPE.equals(detectionType) || DetectorTrigger.RULES_DETECTION_TYPE.equals(detectionType))) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "Trigger [%s] has unsupported detection type [%s]", trigger.getName(), detectionType)); } } } diff --git a/src/main/java/org/opensearch/securityanalytics/resthandler/RestTestS3ConnectionAction.java b/src/main/java/org/opensearch/securityanalytics/resthandler/RestTestS3ConnectionAction.java new file mode 100644 index 000000000..2748d837d --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/resthandler/RestTestS3ConnectionAction.java @@ -0,0 +1,52 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.resthandler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.node.NodeClient; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.securityanalytics.action.TestS3ConnectionAction; +import org.opensearch.securityanalytics.action.TestS3ConnectionRequest; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.TEST_S3_CONNECTION_URI; + +public class RestTestS3ConnectionAction extends BaseRestHandler { + private static final Logger log = LogManager.getLogger(RestTestS3ConnectionAction.class); + + + @Override + public String getName() { + return "test_connection_s3"; + } + + @Override + public List routes() { + return List.of( + new Route(RestRequest.Method.POST, TEST_S3_CONNECTION_URI) + ); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + log.debug(String.format(Locale.getDefault(), "%s %s", request.method(), TEST_S3_CONNECTION_URI)); + + XContentParser xcp = request.contentParser(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + + TestS3ConnectionRequest testRequest = TestS3ConnectionRequest.parse(xcp); + + return channel -> client.execute(TestS3ConnectionAction.INSTANCE, testRequest, new RestToXContentListener<>(channel)); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCConnectorFactory.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCConnectorFactory.java new file mode 100644 index 000000000..7c05f0b57 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCConnectorFactory.java @@ -0,0 +1,57 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.services; + +import com.amazonaws.services.s3.AmazonS3; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.securityanalytics.commons.connector.Connector; +import org.opensearch.securityanalytics.commons.connector.S3Connector; +import org.opensearch.securityanalytics.commons.connector.codec.InputCodec; +import org.opensearch.securityanalytics.commons.connector.factory.InputCodecFactory; +import org.opensearch.securityanalytics.commons.connector.factory.S3ClientFactory; +import org.opensearch.securityanalytics.commons.connector.model.S3ConnectorConfig; +import org.opensearch.securityanalytics.commons.factory.UnaryParameterCachingFactory; +import org.opensearch.securityanalytics.commons.model.FeedConfiguration; +import org.opensearch.securityanalytics.commons.model.FeedLocation; +import org.opensearch.securityanalytics.commons.model.STIX2; +import software.amazon.awssdk.services.s3.S3Client; + +import java.util.List; + +public class STIX2IOCConnectorFactory extends UnaryParameterCachingFactory> { + private static final Logger logger = LogManager.getLogger(STIX2IOCConnectorFactory.class); + private final InputCodecFactory inputCodecFactory; + private final S3ClientFactory s3ClientFactory; + + public STIX2IOCConnectorFactory(final InputCodecFactory inputCodecFactory, final S3ClientFactory s3ClientFactory) { + this.inputCodecFactory = inputCodecFactory; + this.s3ClientFactory = s3ClientFactory; + } + + protected Connector doCreate(FeedConfiguration feedConfiguration) { + final FeedLocation feedLocation = FeedLocation.fromFeedConfiguration(feedConfiguration); + logger.debug("FeedLocation: {}", feedLocation); + switch(feedLocation) { + case S3: return createS3Connector(feedConfiguration); + default: throw new IllegalArgumentException("Unsupported feedLocation: " + feedLocation); + } + } + + private S3Connector createS3Connector(final FeedConfiguration feedConfiguration) { + final S3ConnectorConfig s3ConnectorConfig = feedConfiguration.getS3ConnectorConfig(); + final S3Client s3Client = s3ClientFactory.create(s3ConnectorConfig.getRoleArn(), s3ConnectorConfig.getRegion()); + final InputCodec inputCodec = inputCodecFactory.create(feedConfiguration.getIocSchema().getModelClass(), feedConfiguration.getInputCodecSchema()); + return new S3Connector<>(s3ConnectorConfig, s3Client, inputCodec); + } + + public S3Connector createAmazonS3Connector(final FeedConfiguration feedConfiguration, List clusterTuple) { + final S3ConnectorConfig s3ConnectorConfig = feedConfiguration.getS3ConnectorConfig(); + final AmazonS3 s3Client = s3ClientFactory.createAmazonS3(s3ConnectorConfig.getRoleArn(), s3ConnectorConfig.getRegion(), clusterTuple); + final InputCodec inputCodec = inputCodecFactory.create(feedConfiguration.getIocSchema().getModelClass(), feedConfiguration.getInputCodecSchema()); + return new S3Connector<>(s3ConnectorConfig, s3Client, inputCodec); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCConsumer.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCConsumer.java new file mode 100644 index 000000000..9808b4387 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCConsumer.java @@ -0,0 +1,91 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.services; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.commons.model.IOC; +import org.opensearch.securityanalytics.commons.model.STIX2; +import org.opensearch.securityanalytics.commons.model.UpdateAction; +import org.opensearch.securityanalytics.commons.model.UpdateType; +import org.opensearch.securityanalytics.model.STIX2IOC; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +public class STIX2IOCConsumer implements Consumer { + private final Logger log = LogManager.getLogger(STIX2IOCConsumer.class); + private final LinkedBlockingQueue queue; + private final STIX2IOCFeedStore feedStore; + private final UpdateType updateType; + + public STIX2IOCConsumer(final int batchSize, final STIX2IOCFeedStore feedStore, final UpdateType updateType) { + this.queue = new LinkedBlockingQueue<>(batchSize); + this.feedStore = feedStore; + this.updateType = updateType; + } + + @Override + public void accept(final STIX2 ioc) { + STIX2IOC stix2IOC = new STIX2IOC( + ioc, + feedStore.getSaTifSourceConfig().getId(), + feedStore.getSaTifSourceConfig().getName() + ); + + // If the IOC received is not a type listed for the config, do not add it to the queue + if (!feedStore.getSaTifSourceConfig().getIocTypes().contains(stix2IOC.getType().toString())) { + log.error("{} is not a supported Ioc type for tif source config {}. Skipping IOC {}: of type {} value {}", + stix2IOC.getType().toString(), feedStore.getSaTifSourceConfig().getId(), + stix2IOC.getId(), stix2IOC.getType(), stix2IOC.getValue() + ); + return; + } + + if (queue.offer(stix2IOC)) { + return; + } + + flushIOCs(); + queue.offer(stix2IOC); + } + + public void flushIOCs() { + if (queue.isEmpty()) { + throw new OpenSearchStatusException("No compatible Iocs were downloaded for config " + feedStore.getSaTifSourceConfig().getName(), RestStatus.BAD_REQUEST); + } + + final List iocsToFlush = new ArrayList<>(queue.size()); + queue.drainTo(iocsToFlush); + + final Map iocToActions = buildIOCToActions(iocsToFlush); + feedStore.storeIOCs(iocToActions); + } + + private Map buildIOCToActions(final List iocs) { + switch (updateType) { + case REPLACE: return buildReplaceActions(iocs); + case DELTA: return buildDeltaActions(iocs); + default: throw new IllegalArgumentException("Invalid update type: " + updateType); + } + } + + private Map buildReplaceActions(final List iocs) { + return iocs.stream() + .map(ioc -> Map.entry(ioc, UpdateAction.UPSERT)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private Map buildDeltaActions(final List iocs) { + throw new UnsupportedOperationException("Delta update type is not yet supported"); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java new file mode 100644 index 000000000..695a9d65a --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java @@ -0,0 +1,269 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.services; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchException; +import org.opensearch.ResourceAlreadyExistsException; +import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.StepListener; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.io.Streams; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.securityanalytics.commons.model.IOC; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.commons.model.UpdateAction; +import org.opensearch.securityanalytics.commons.store.FeedStore; +import org.opensearch.securityanalytics.model.STIX2IOC; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; +import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.transport.RemoteTransportException; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.UUID; + +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; + +public class STIX2IOCFeedStore implements FeedStore { + public static final String IOC_INDEX_NAME_BASE = ".opensearch-sap-iocs"; + public static final String IOC_ALL_INDEX_PATTERN = IOC_INDEX_NAME_BASE + "-*"; + public static final String IOC_FEED_ID_PLACEHOLDER = "FEED_ID"; + public static final String IOC_INDEX_NAME_TEMPLATE = IOC_INDEX_NAME_BASE + "-" + IOC_FEED_ID_PLACEHOLDER; + public static final String IOC_ALL_INDEX_PATTERN_BY_ID = IOC_INDEX_NAME_TEMPLATE + "-*"; + public static final String IOC_TIME_PLACEHOLDER = "TIME"; + public static final String IOC_INDEX_PATTERN = IOC_INDEX_NAME_TEMPLATE + "-" + IOC_TIME_PLACEHOLDER; + + private final Logger log = LogManager.getLogger(STIX2IOCFeedStore.class); + private final String newActiveIndex; + + Instant startTime = Instant.now(); + + private Client client; + private ClusterService clusterService; + private SATIFSourceConfig saTifSourceConfig; + private ActionListener baseListener; + private Integer batchSize; + + public STIX2IOCFeedStore( + Client client, + ClusterService clusterService, + SATIFSourceConfig saTifSourceConfig, + ActionListener listener) { + super(); + this.client = client; + this.clusterService = clusterService; + this.saTifSourceConfig = saTifSourceConfig; + this.baseListener = listener; + batchSize = clusterService.getClusterSettings().get(SecurityAnalyticsSettings.BATCH_SIZE); + newActiveIndex = getNewActiveIndex(saTifSourceConfig.getId()); + } + + @Override + public void storeIOCs(Map actionToIOCs) { + Map> iocsSortedByAction = new HashMap<>(); + actionToIOCs.forEach((key, value) -> { + if (key.getClass() != STIX2IOC.class) { + throw new IllegalArgumentException("Only supports STIX2-formatted IOCs."); + } else { + iocsSortedByAction.putIfAbsent(value, new ArrayList<>()); + iocsSortedByAction.get(value).add((STIX2IOC) key); + } + }); + + for (Map.Entry> entry : iocsSortedByAction.entrySet()) { + switch (entry.getKey()) { + case DELETE: + break; + case UPSERT: + try { + indexIocs(entry.getValue()); + } catch (IOException e) { + baseListener.onFailure(new RuntimeException(e)); + } + break; + default: + baseListener.onFailure(new IllegalArgumentException("Unsupported action.")); + } + } + } + + public void indexIocs(List iocs) throws IOException { + StepListener initSourceConfigIndexesListener = new StepListener<>(); + initSourceConfigIndexes(initSourceConfigIndexesListener); + initSourceConfigIndexesListener.whenComplete(r -> { + bulkIndexIocs(iocs, newActiveIndex); + }, e -> { + log.error("Failed to init source config indexes"); + baseListener.onFailure(e); + }); + + } + + private void bulkIndexIocs(List iocs, String activeIndex) throws IOException { + if (iocs.isEmpty()) { + long duration = Duration.between(startTime, Instant.now()).toMillis(); + STIX2IOCFetchService.STIX2IOCFetchResponse output = new STIX2IOCFetchService.STIX2IOCFetchResponse(Collections.emptyList(), duration); + baseListener.onResponse(output); + } + + List bulkRequestList = new ArrayList<>(); + BulkRequest bulkRequest = new BulkRequest(); + for (STIX2IOC ioc : iocs) { + IndexRequest indexRequest = new IndexRequest(activeIndex) + .id(StringUtils.isBlank(ioc.getId()) ? UUID.randomUUID().toString() : ioc.getId()) + .opType(DocWriteRequest.OpType.INDEX) + .source(ioc.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)); + bulkRequest.add(indexRequest); + + if (bulkRequest.requests().size() == batchSize) { + bulkRequestList.add(bulkRequest); + bulkRequest = new BulkRequest(); + } + } + + if (!bulkRequest.requests().isEmpty()) bulkRequestList.add(bulkRequest); + if (!bulkRequestList.isEmpty()) bulkRequestList.get(bulkRequestList.size() - 1).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + GroupedActionListener bulkResponseListener = new GroupedActionListener<>(ActionListener.wrap(bulkResponses -> { + int idx = 0; + for (BulkResponse response : bulkResponses) { + BulkRequest request = bulkRequestList.get(idx); + if (response.hasFailures()) { + throw new OpenSearchException( + "Error occurred while ingesting IOCs to {} with an error {}", + StringUtils.join(request.getIndices()), + response.buildFailureMessage() + ); + } + idx++; + } + long duration = Duration.between(startTime, Instant.now()).toMillis(); + STIX2IOCFetchService.STIX2IOCFetchResponse output = new STIX2IOCFetchService.STIX2IOCFetchResponse(iocs, duration); + baseListener.onResponse(output); + }, e -> { + log.error("Failed to index IOCs for config {}", saTifSourceConfig.getId(), e); + baseListener.onFailure(e); + }), bulkRequestList.size()); + + for (BulkRequest req : bulkRequestList) { + try { + StashedThreadContext.run(client, () -> client.bulk(req, bulkResponseListener)); + } catch (OpenSearchException e) { + log.error("Failed to save IOCs for config {}", saTifSourceConfig.getId(), e); + baseListener.onFailure(e); + } + } + } + + public static String getAllIocIndexPatternById(String sourceConfigId) { + return IOC_ALL_INDEX_PATTERN_BY_ID.replace(IOC_FEED_ID_PLACEHOLDER, sourceConfigId.toLowerCase(Locale.ROOT)); + } + + public static String getNewActiveIndex(String sourceConfigId) { + return IOC_INDEX_PATTERN + .replace(IOC_FEED_ID_PLACEHOLDER, sourceConfigId.toLowerCase(Locale.ROOT)) + .replace(IOC_TIME_PLACEHOLDER, Long.toString(Instant.now().toEpochMilli())); + } + + public String iocIndexMapping() { + String iocMappingFile = "mappings/stix2_ioc_mapping.json"; + try (InputStream is = getClass().getClassLoader().getResourceAsStream(iocMappingFile)) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + Streams.copy(is, out); + return out.toString(StandardCharsets.UTF_8); + } catch (Exception e) { + throw new IllegalStateException("Failed to load stix2_ioc_mapping.json file [" + iocMappingFile + "]", e); + } + } + + public SATIFSourceConfig getSaTifSourceConfig() { + return saTifSourceConfig; + } + + private void initSourceConfigIndexes(StepListener stepListener) { + String iocIndexPattern = getAllIocIndexPatternById(saTifSourceConfig.getId()); + initFeedIndex(newActiveIndex, ActionListener.wrap( + r -> { + // reset the store configs + if (saTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) { + ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails().clear(); + } + + // recreate the store configs + saTifSourceConfig.getIocTypes().forEach(type -> { + if (saTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) { + DefaultIocStoreConfig.IocToIndexDetails iocToIndexDetails = + new DefaultIocStoreConfig.IocToIndexDetails(new IOCType(type), iocIndexPattern, newActiveIndex); + ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails().add(iocToIndexDetails); + } + }); + stepListener.onResponse(null); + }, e-> { + log.error("Failed to initialize the IOC index and save the IOCs", e); + stepListener.onFailure(e); + } + )); + } + + private void initFeedIndex(String feedIndexName, ActionListener listener) { + if (!clusterService.state().routingTable().hasIndex(newActiveIndex)) { + var indexRequest = new CreateIndexRequest(feedIndexName) + .mapping(iocIndexMapping()) + .settings(Settings.builder() + .put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) + .build() + ); + client.admin().indices().create(indexRequest, ActionListener.wrap( + r -> { + log.info("Created system index {}", feedIndexName); + listener.onResponse(r); + }, + e -> { + if (e instanceof ResourceAlreadyExistsException || (e instanceof RemoteTransportException && e.getCause() instanceof ResourceAlreadyExistsException)) { + log.debug("index {} already exist", feedIndexName); + listener.onResponse(null); + return; + } + log.error("Failed to create system index {}", feedIndexName); + listener.onFailure(e); + } + )); + } else { + listener.onResponse(null); + } + } +} + diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java new file mode 100644 index 000000000..31f4c6f2a --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java @@ -0,0 +1,446 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.services; + +import com.amazonaws.AmazonServiceException; +import com.amazonaws.SdkClientException; +import com.fasterxml.jackson.databind.RuntimeJsonMappingException; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchException; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.securityanalytics.action.TestS3ConnectionResponse; +import org.opensearch.securityanalytics.commons.connector.Connector; +import org.opensearch.securityanalytics.commons.connector.S3Connector; +import org.opensearch.securityanalytics.commons.connector.exceptions.ConnectorParsingException; +import org.opensearch.securityanalytics.commons.connector.factory.InputCodecFactory; +import org.opensearch.securityanalytics.commons.connector.factory.S3ClientFactory; +import org.opensearch.securityanalytics.commons.connector.factory.StsAssumeRoleCredentialsProviderFactory; +import org.opensearch.securityanalytics.commons.connector.factory.StsClientFactory; +import org.opensearch.securityanalytics.commons.connector.model.InputCodecSchema; +import org.opensearch.securityanalytics.commons.connector.model.S3ConnectorConfig; +import org.opensearch.securityanalytics.commons.model.FeedConfiguration; +import org.opensearch.securityanalytics.commons.model.IOCSchema; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.commons.model.STIX2; +import org.opensearch.securityanalytics.commons.model.UpdateType; +import org.opensearch.securityanalytics.model.STIX2IOC; +import org.opensearch.securityanalytics.model.STIX2IOCDto; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.model.S3Source; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.util.ThreatIntelFeedParser; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.sts.model.StsException; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService.isValidIp; + +/** + * IOC Service implements operations that interact with retrieving IOCs from data sources, + * parsing them into threat intel data models (i.e., [IOC]), and ingesting them to system indexes. + */ +public class STIX2IOCFetchService { + private final Logger log = LogManager.getLogger(STIX2IOCFetchService.class); + private final String ENDPOINT_CONFIG_PATH = "/threatIntelFeed/internalAuthEndpoint.txt"; + + public final String REGION_REGEX = "^.{1,20}$"; + public final String ROLE_ARN_REGEX = "^arn:aws:iam::\\d{12}:role/[\\w+=,.@-]{1,64}$"; + + private Client client; + private ClusterService clusterService; + private STIX2IOCConnectorFactory connectorFactory; + private S3ClientFactory s3ClientFactory; + + private Integer batchSize; + private String internalAuthEndpoint = ""; + + public STIX2IOCFetchService(Client client, ClusterService clusterService) { + this.client = client; + this.clusterService = clusterService; + this.internalAuthEndpoint = getEndpoint(); + + StsAssumeRoleCredentialsProviderFactory factory = + new StsAssumeRoleCredentialsProviderFactory(new StsClientFactory()); + s3ClientFactory = new S3ClientFactory(factory, internalAuthEndpoint); + connectorFactory = new STIX2IOCConnectorFactory(new InputCodecFactory(), s3ClientFactory); + batchSize = clusterService.getClusterSettings().get(SecurityAnalyticsSettings.BATCH_SIZE); + } + + /** + * Method takes in and calls method to rollover and bulk index a list of STIX2IOCs + * + * @param saTifSourceConfig + * @param stix2IOCList + * @param listener + */ + public void onlyIndexIocs(SATIFSourceConfig saTifSourceConfig, + List stix2IOCList, + ActionListener listener) { + STIX2IOCFeedStore feedStore = new STIX2IOCFeedStore(client, clusterService, saTifSourceConfig, listener); + Instant startTime = Instant.now(); + Instant endTime; + Exception exception = null; + RestStatus restStatus = null; + try { + log.info("Started IOC index step at {}.", startTime); + feedStore.indexIocs(stix2IOCList); + } catch (IllegalArgumentException e) { + exception = e; + restStatus = RestStatus.BAD_REQUEST; + } catch (OpenSearchException e) { + exception = e; + restStatus = e.status(); + } catch (Exception e) { + exception = e; + restStatus = RestStatus.INTERNAL_SERVER_ERROR; + } + endTime = Instant.now(); + long took = Duration.between(startTime, endTime).toMillis(); + + if (exception != null && restStatus != null) { + String errorText = getErrorText(saTifSourceConfig, "index", took); + log.error(errorText, exception); + listener.onFailure(new SecurityAnalyticsException(errorText, restStatus, exception)); + } else { + log.info("IOC index step took {} milliseconds.", took); + } + } + + public void downloadAndIndexIOCs(SATIFSourceConfig saTifSourceConfig, ActionListener listener) { + S3ConnectorConfig s3ConnectorConfig; + try { + s3ConnectorConfig = constructS3ConnectorConfig(saTifSourceConfig); + } catch (SecurityAnalyticsException e) { + listener.onFailure(e); + return; + } + + Connector s3Connector = constructS3Connector(s3ConnectorConfig); + STIX2IOCFeedStore feedStore = new STIX2IOCFeedStore(client, clusterService, saTifSourceConfig, listener); + STIX2IOCConsumer consumer = new STIX2IOCConsumer(batchSize, feedStore, UpdateType.REPLACE); + + Instant startTime = Instant.now(); + Instant endTime; + Exception exception = null; + RestStatus restStatus = null; + try { + log.info("Started IOC download step at {}.", startTime); + s3Connector.load(consumer); + } catch (IllegalArgumentException | ConnectorParsingException | RuntimeJsonMappingException e) { + exception = e; + restStatus = RestStatus.BAD_REQUEST; + } catch (StsException | S3Exception e) { + exception = e; + restStatus = RestStatus.fromCode(e.statusCode()); + } catch (AmazonServiceException e) { + exception = e; + restStatus = RestStatus.fromCode(e.getStatusCode()); + } catch (SdkException | SdkClientException e) { + // SdkException is a RunTimeException that doesn't have a status code. + // Logging the full exception, and providing generic response as output. + exception = e; + restStatus = RestStatus.FORBIDDEN; + } catch (Exception e) { + exception = e; + restStatus = RestStatus.INTERNAL_SERVER_ERROR; + } + endTime = Instant.now(); + long took = Duration.between(startTime, endTime).toMillis(); + + if (exception != null && restStatus != null) { + String errorText = getErrorText(saTifSourceConfig, "download", took); + log.error(errorText, exception); + listener.onFailure(new SecurityAnalyticsException(errorText, restStatus, exception)); + return; + } else { + log.info("IOC download step took {} milliseconds.", took); + } + + startTime = Instant.now(); + try { + log.info("Started IOC flush at {}.", startTime); + consumer.flushIOCs(); + } catch (IllegalArgumentException e) { + exception = e; + restStatus = RestStatus.BAD_REQUEST; + } catch (OpenSearchException e) { + exception = e; + restStatus = e.status(); + } catch (Exception e) { + exception = e; + restStatus = RestStatus.INTERNAL_SERVER_ERROR; + } + endTime = Instant.now(); + took = Duration.between(startTime, endTime).toMillis(); + + if (exception != null && restStatus != null) { + String errorText = getErrorText(saTifSourceConfig, "index", took); + log.error(errorText, exception); + listener.onFailure(new SecurityAnalyticsException(errorText, restStatus, exception)); + } else { + log.info("IOC flush step took {} milliseconds.", took); + } + } + + public void testS3Connection(S3ConnectorConfig s3ConnectorConfig, ActionListener listener) { + if (internalAuthEndpoint.isEmpty()) { + testS3ClientConnection(s3ConnectorConfig, listener); + } else { + testAmazonS3Connection(s3ConnectorConfig, listener); + } + } + + private void testS3ClientConnection(S3ConnectorConfig s3ConnectorConfig, ActionListener listener) { + try { + S3Connector connector = (S3Connector) constructS3Connector(s3ConnectorConfig); + HeadObjectResponse response = connector.testS3Connection(s3ConnectorConfig); + listener.onResponse(new TestS3ConnectionResponse(RestStatus.fromCode(response.sdkHttpResponse().statusCode()), "")); + } catch (NoSuchKeyException noSuchKeyException) { + log.error("S3Client connection test failed with NoSuchKeyException: ", noSuchKeyException); + listener.onResponse(new TestS3ConnectionResponse(RestStatus.fromCode(noSuchKeyException.statusCode()), noSuchKeyException.awsErrorDetails().errorMessage())); + } catch (S3Exception s3Exception) { + log.error("S3Client connection test failed with S3Exception: ", s3Exception); + listener.onResponse(new TestS3ConnectionResponse(RestStatus.fromCode(s3Exception.statusCode()), "Resource not found.")); + } catch (StsException stsException) { + log.error("S3Client connection test failed with StsException: ", stsException); + listener.onResponse(new TestS3ConnectionResponse(RestStatus.fromCode(stsException.statusCode()), stsException.awsErrorDetails().errorMessage())); + } catch (SdkException sdkException) { + // SdkException is a RunTimeException that doesn't have a status code. + // Logging the full exception, and providing generic response as output. + log.error("S3Client connection test failed with SdkException: ", sdkException); + listener.onResponse(new TestS3ConnectionResponse(RestStatus.FORBIDDEN, "Resource not found.")); + } catch (Exception e) { + log.error("S3Client connection test failed with error: ", e); + listener.onFailure(SecurityAnalyticsException.wrap(e)); + } + } + + private void testAmazonS3Connection(S3ConnectorConfig s3ConnectorConfig, ActionListener listener) { + try { + S3Connector connector = (S3Connector) constructS3Connector(s3ConnectorConfig); + boolean response = connector.testAmazonS3Connection(s3ConnectorConfig); + listener.onResponse(new TestS3ConnectionResponse(response ? RestStatus.OK : RestStatus.FORBIDDEN, "")); + } catch (AmazonServiceException e) { + log.error("AmazonS3 connection test failed with AmazonServiceException: ", e); + listener.onResponse(new TestS3ConnectionResponse(RestStatus.fromCode(e.getStatusCode()), e.getErrorMessage())); + } catch (SdkClientException e) { + // SdkException is a RunTimeException that doesn't have a status code. + // Logging the full exception, and providing generic response as output. + log.error("AmazonS3 connection test failed with SdkClientException: ", e); + listener.onResponse(new TestS3ConnectionResponse(RestStatus.FORBIDDEN, "Resource not found.")); + } catch (Exception e) { + log.error("AmazonS3 connection test failed with error: ", e); + listener.onFailure(SecurityAnalyticsException.wrap(e)); + } + } + + private Connector constructS3Connector(S3ConnectorConfig s3ConnectorConfig) { + FeedConfiguration feedConfiguration = new FeedConfiguration(IOCSchema.STIX2, InputCodecSchema.ND_JSON, s3ConnectorConfig); + if (internalAuthEndpoint.isEmpty()) { + return constructS3ClientConnector(feedConfiguration); + } else { + return constructAmazonS3Connector(feedConfiguration); + } + } + + private Connector constructS3ClientConnector(FeedConfiguration feedConfiguration) { + return connectorFactory.doCreate(feedConfiguration); + } + + private Connector constructAmazonS3Connector(FeedConfiguration feedConfiguration) { + List clusterTuple = List.of(clusterService.getClusterName().value().split(":")); + return connectorFactory.createAmazonS3Connector(feedConfiguration, clusterTuple); + } + + private S3ConnectorConfig constructS3ConnectorConfig(SATIFSourceConfig saTifSourceConfig) { + S3ConnectorConfig s3ConnectorConfig = new S3ConnectorConfig( + ((S3Source) saTifSourceConfig.getSource()).getBucketName(), + ((S3Source) saTifSourceConfig.getSource()).getObjectKey(), + ((S3Source) saTifSourceConfig.getSource()).getRegion(), + ((S3Source) saTifSourceConfig.getSource()).getRoleArn() + ); + validateS3ConnectorConfig(s3ConnectorConfig); + return s3ConnectorConfig; + } + + private void validateS3ConnectorConfig(S3ConnectorConfig s3ConnectorConfig) { + if (s3ConnectorConfig.getRoleArn() == null || !s3ConnectorConfig.getRoleArn().matches(ROLE_ARN_REGEX)) { + throw new SecurityAnalyticsException("Role arn is empty or malformed.", RestStatus.BAD_REQUEST, new IllegalArgumentException()); + } + + if (s3ConnectorConfig.getRegion() == null || !s3ConnectorConfig.getRegion().matches(REGION_REGEX)) { + throw new SecurityAnalyticsException("Region is empty or malformed.", RestStatus.BAD_REQUEST, new IllegalArgumentException()); + } + } + + private String getEndpoint() { + try { + try (InputStream is = TIFJobParameterService.class.getResourceAsStream(ENDPOINT_CONFIG_PATH)) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + return reader.lines().map(String::trim).collect(Collectors.joining()); + } + } + } catch (Exception e) { + log.debug(String.format("Resource file [%s] doesn't exist.", ENDPOINT_CONFIG_PATH)); + } + return ""; + } + + public void downloadFromUrlAndIndexIOCs(SATIFSourceConfig saTifSourceConfig, ActionListener listener) { + UrlDownloadSource source = (UrlDownloadSource) saTifSourceConfig.getSource(); + switch (source.getFeedFormat()) { // todo add check to stop user from creating url type config from rest api. only internal allowed + case "csv": + try (CSVParser reader = ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(source.getUrl())) { + CSVParser noHeaderReader = ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(source.getUrl()); + boolean notFound = true; + + while (notFound) { + CSVRecord hasHeaderRecord = reader.iterator().next(); + + //if we want to skip this line and keep iterating + if ((hasHeaderRecord.values().length == 1 && "".equals(hasHeaderRecord.values()[0])) || hasHeaderRecord.get(0).charAt(0) == '#' || hasHeaderRecord.get(0).charAt(0) == ' ') { + noHeaderReader.iterator().next(); + } else { // we found the first line that contains information + notFound = false; + } + } + if (source.hasCsvHeader()) { + parseAndSaveThreatIntelFeedDataCSV(reader.iterator(), saTifSourceConfig, listener); + } else { + parseAndSaveThreatIntelFeedDataCSV(noHeaderReader.iterator(), saTifSourceConfig, listener); + } + } catch (Exception e) { + log.error("Failed to download the IoCs in CSV format for source " + saTifSourceConfig.getId()); + listener.onFailure(SecurityAnalyticsException.wrap(e)); + return; + } + break; + default: + log.error("unsupported feed format for url download:" + source.getFeedFormat()); + listener.onFailure(SecurityAnalyticsException.wrap(new UnsupportedOperationException("unsupported feed format for url download:" + source.getFeedFormat()))); + } + } + + private void parseAndSaveThreatIntelFeedDataCSV(Iterator iterator, SATIFSourceConfig saTifSourceConfig, ActionListener listener) throws IOException { + List bulkRequestList = new ArrayList<>(); + + UrlDownloadSource source = (UrlDownloadSource) saTifSourceConfig.getSource(); + List iocs = new ArrayList<>(); + while (iterator.hasNext()) { + CSVRecord record = iterator.next(); + String iocType = saTifSourceConfig.getIocTypes().stream().findFirst().orElse(null); + Integer colNum = source.getCsvIocValueColumnNo(); + String iocValue = record.values()[colNum].split(" ")[0]; + if (iocType.equalsIgnoreCase(IOCType.IPV4_TYPE) && !isValidIp(iocValue)) { + log.info("Invalid IP address, skipping this ioc record: {}", iocValue); + continue; + } + Instant now = Instant.now(); + STIX2IOC stix2IOC = new STIX2IOC( + UUID.randomUUID().toString(), + UUID.randomUUID().toString(), + iocType == null ? new IOCType(IOCType.IPV4_TYPE) : new IOCType(iocType), + iocValue, + "high", + now, + now, + "", + Collections.emptyList(), + "", + saTifSourceConfig.getId(), + saTifSourceConfig.getName(), + STIX2IOC.NO_VERSION + ); + iocs.add(stix2IOC); + } + STIX2IOCFeedStore feedStore = new STIX2IOCFeedStore(client, clusterService, saTifSourceConfig, listener); + feedStore.indexIocs(iocs); + } + + /** + * Helper function for generating error message text. + * @param saTifSourceConfig The config for which IOCs are being downloaded/indexed. + * @param action The action that was being taken when the error occurred; e.g., "download", or "index". + * @param duration The amount of time, in milliseconds, it took for the action to fail. + * @return The error message text. + */ + private String getErrorText(SATIFSourceConfig saTifSourceConfig, String action, long duration) { + return String.format( + "Failed to %s IOCs from source config '%s' with ID %s after %s milliseconds: ", + action, + saTifSourceConfig.getName(), + saTifSourceConfig.getId(), + duration + ); + } + + public static class STIX2IOCFetchResponse extends ActionResponse implements ToXContentObject { + public static String IOCS_FIELD = "iocs"; + public static String TOTAL_FIELD = "total"; + public static String DURATION_FIELD = "took"; + private List iocs = new ArrayList<>(); + private long duration; // In milliseconds + + public STIX2IOCFetchResponse(List iocs, long duration) { + super(); + iocs.forEach(ioc -> this.iocs.add(new STIX2IOCDto(ioc))); + this.duration = duration; + } + + public STIX2IOCFetchResponse(StreamInput sin) throws IOException { + this(sin.readList(STIX2IOC::new), sin.readLong()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeList(iocs); + out.writeLong(duration); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field(TOTAL_FIELD, iocs.size()) + .field(DURATION_FIELD, duration) + .endObject(); + } + + public List getIocs() { + return iocs; + } + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java index fefe7c288..cc11fe36d 100644 --- a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java +++ b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java @@ -12,6 +12,8 @@ public class SecurityAnalyticsSettings { public static final String CORRELATION_INDEX = "index.correlation"; + public static final int minSystemIndexReplicas = 0; + public static final int maxSystemIndexReplicas = 20; public static Setting INDEX_TIMEOUT = Setting.positiveTimeSetting("plugins.security_analytics.index_timeout", TimeValue.timeValueSeconds(60), @@ -31,6 +33,12 @@ public class SecurityAnalyticsSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ); + public static final Setting IOC_FINDING_HISTORY_ENABLED = Setting.boolSetting( + "plugins.security_analytics.ioc_finding_enabled", + true, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + public static final Setting ALERT_HISTORY_ROLLOVER_PERIOD = Setting.positiveTimeSetting( "plugins.security_analytics.alert_history_rollover_period", TimeValue.timeValueHours(12), @@ -49,6 +57,12 @@ public class SecurityAnalyticsSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ); + public static final Setting IOC_FINDING_HISTORY_ROLLOVER_PERIOD = Setting.positiveTimeSetting( + "plugins.security_analytics.ioc_finding_history_rollover_period", + TimeValue.timeValueHours(12), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + public static final Setting ALERT_HISTORY_INDEX_MAX_AGE = Setting.positiveTimeSetting( "plugins.security_analytics.alert_history_max_age", new TimeValue(30, TimeUnit.DAYS), @@ -67,6 +81,12 @@ public class SecurityAnalyticsSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ); + public static final Setting IOC_FINDING_HISTORY_INDEX_MAX_AGE = Setting.positiveTimeSetting( + "plugins.security_analytics.ioc_finding_history_max_age", + new TimeValue(30, TimeUnit.DAYS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + public static final Setting ALERT_HISTORY_MAX_DOCS = Setting.longSetting( "plugins.security_analytics.alert_history_max_docs", 1000L, @@ -88,6 +108,13 @@ public class SecurityAnalyticsSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ); + public static final Setting IOC_FINDING_HISTORY_MAX_DOCS = Setting.longSetting( + "plugins.security_analytics.ioc_finding_history_max_docs", + 1000L, + 0L, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + public static final Setting ALERT_HISTORY_RETENTION_PERIOD = Setting.positiveTimeSetting( "plugins.security_analytics.alert_history_retention_period", new TimeValue(60, TimeUnit.DAYS), @@ -106,6 +133,12 @@ public class SecurityAnalyticsSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ); + public static final Setting IOC_FINDING_HISTORY_RETENTION_PERIOD = Setting.positiveTimeSetting( + "plugins.security_analytics.ioc_finding_history_retention_period", + new TimeValue(60, TimeUnit.DAYS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + public static final Setting REQUEST_TIMEOUT = Setting.positiveTimeSetting( "plugins.security_analytics.request_timeout", TimeValue.timeValueSeconds(10), @@ -191,4 +224,35 @@ public static final List> settings() { return List.of(BATCH_SIZE, THREAT_INTEL_TIMEOUT, TIF_UPDATE_INTERVAL); } + // Threat Intel IOC Settings + public static final Setting IOC_INDEX_RETENTION_PERIOD = Setting.timeSetting( + "plugins.security_analytics.ioc.index_retention_period", + new TimeValue(30, TimeUnit.DAYS), + new TimeValue(1, TimeUnit.DAYS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting IOC_MAX_INDICES_PER_INDEX_PATTERN = Setting.intSetting( + "plugins.security_analytics.ioc.max_indices_per_alias", + 30, + 1, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + /** + * Maximum terms in Terms query search query submitted during ioc scan + */ + public static final Setting IOC_SCAN_MAX_TERMS_COUNT = Setting.intSetting( + "plugins.security_analytics.ioc.scan_max_terms_count", + 65536, + 1, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting ENABLE_DETECTORS_WITH_DEDICATED_QUERY_INDICES = Setting.boolSetting( + "plugins.security_analytics.enable_detectors_with_dedicated_query_indices", + true, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsAction.java new file mode 100644 index 000000000..77182371f --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsAction.java @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionType; + +public class GetIocFindingsAction extends ActionType { + + public static final GetIocFindingsAction INSTANCE = new GetIocFindingsAction(); + public static final String NAME = "cluster:admin/opensearch/securityanalytics/threatintel/iocs/findings/get"; + + public GetIocFindingsAction() { + super(NAME, GetIocFindingsResponse::new); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsRequest.java new file mode 100644 index 000000000..1395cff1e --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsRequest.java @@ -0,0 +1,91 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.ValidateActions; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Locale; + +public class GetIocFindingsRequest extends ActionRequest { + + private List findingIds; + + private List iocIds; + + private Instant startTime; + + private Instant endTime; + + private Table table; + + public GetIocFindingsRequest(StreamInput sin) throws IOException { + this( + sin.readOptionalStringList(), + sin.readOptionalStringList(), + sin.readOptionalInstant(), + sin.readOptionalInstant(), + Table.readFrom(sin) + ); + } + + public GetIocFindingsRequest(List findingIds, + List iocIds, + Instant startTime, + Instant endTime, + Table table) { + this.findingIds = findingIds; + this.iocIds = iocIds; + this.startTime = startTime; + this.endTime = endTime; + this.table = table; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (startTime != null && endTime != null && startTime.isAfter(endTime)) { + validationException = ValidateActions.addValidationError(String.format(Locale.getDefault(), + "startTime should be less than endTime"), validationException); + } + return validationException; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalStringCollection(findingIds); + out.writeOptionalStringCollection(iocIds); + out.writeOptionalInstant(startTime); + out.writeOptionalInstant(endTime); + table.writeTo(out); + } + + public List getFindingIds() { + return findingIds; + } + + public List getIocIds() { + return iocIds; + } + + public Instant getStartTime() { + return startTime; + } + + public Instant getEndTime() { + return endTime; + } + + public Table getTable() { + return table; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java new file mode 100644 index 000000000..50ae08dd4 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java @@ -0,0 +1,62 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class GetIocFindingsResponse extends ActionResponse implements ToXContentObject { + + private static final String TOTAL_IOC_FINDINGS_FIELD = "total_findings"; + + private static final String IOC_FINDINGS_FIELD = "ioc_findings"; + + private Integer totalFindings; + + private List iocFindings; + + public GetIocFindingsResponse(Integer totalFindings, List iocFindings) { + super(); + this.totalFindings = totalFindings; + this.iocFindings = iocFindings; + } + + public GetIocFindingsResponse(StreamInput sin) throws IOException { + this( + sin.readInt(), + Collections.unmodifiableList(sin.readList(IocFinding::new)) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(totalFindings); + out.writeCollection(iocFindings); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(TOTAL_IOC_FINDINGS_FIELD, totalFindings) + .field(IOC_FINDINGS_FIELD, iocFindings); + return builder.endObject(); + } + + public Integer getTotalFindings() { + return totalFindings; + } + + public List getIocFindings() { + return iocFindings; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsAction.java new file mode 100644 index 000000000..f9e5bde66 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsAction.java @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionType; + +public class ListIOCsAction extends ActionType { + public static final ListIOCsAction INSTANCE = new ListIOCsAction(); + public static final String NAME = "cluster:admin/opensearch/securityanalytics/threatintel/iocs/list"; + + public ListIOCsAction() { + super(NAME, ListIOCsActionResponse::new); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsActionRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsActionRequest.java new file mode 100644 index 000000000..cb57213b9 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsActionRequest.java @@ -0,0 +1,98 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.ValidateActions; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.securityanalytics.commons.model.IOCType; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; +import java.util.stream.Collectors; + +import static java.util.Collections.emptyList; + +public class ListIOCsActionRequest extends ActionRequest { + + public static final String FEED_IDS_FIELD = "feed_ids"; + public static String SEARCH_FIELD = "search"; + public static String TYPE_FIELD = "ioc_types"; + public static String ALL_TYPES_FILTER = "ALL"; + + private final Table table; + private List types; + private List feedIds; + + public ListIOCsActionRequest(List types, List feedIds, Table table) { + this.table = table; + this.types = types == null + ? emptyList() + : types.stream().map(t -> t.toLowerCase(Locale.ROOT)).collect(Collectors.toList()); + this.feedIds = feedIds == null ? emptyList() : feedIds; + } + + public ListIOCsActionRequest(StreamInput sin) throws IOException { + this( + sin.readOptionalStringList(), // type + sin.readOptionalStringList(), //feedId + Table.readFrom(sin) //table + + ); + } + + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalStringCollection(types); + out.writeOptionalStringCollection(feedIds); + table.writeTo(out); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (table.getStartIndex() < 0) { + validationException = ValidateActions + .addValidationError(String.format("start_index param cannot be a negative number."), validationException); + } else if (table.getSize() < 0 || table.getSize() > 10000) { + validationException = ValidateActions + .addValidationError(String.format("size param must be between 0 and 10,000."), validationException); + } else { + for (String type : types) { + if (!ALL_TYPES_FILTER.equalsIgnoreCase(type)) { + try { + IOCType.fromString(type); + } catch (IllegalArgumentException e) { + validationException = ValidateActions + .addValidationError(String.format("Unrecognized [%s] param.", TYPE_FIELD), validationException); + break; + } + } + } + } + return validationException; + } + + public Table getTable() { + return table; + } + + public List getTypes() { + return types; + } + + public List getFeedIds() { + return feedIds; + } + + public enum SortOrder { + asc, + dsc + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsActionResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsActionResponse.java new file mode 100644 index 000000000..0f142fbf0 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsActionResponse.java @@ -0,0 +1,52 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.securityanalytics.model.DetailedSTIX2IOCDto; +import org.opensearch.securityanalytics.model.STIX2IOCDto; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class ListIOCsActionResponse extends ActionResponse implements ToXContentObject { + public static String TOTAL_HITS_FIELD = "total"; + public static String HITS_FIELD = "iocs"; + + public static ListIOCsActionResponse EMPTY_RESPONSE = new ListIOCsActionResponse(0, Collections.emptyList()); + + private long totalHits; + private List hits; + + public ListIOCsActionResponse(long totalHits, List hits) { + super(); + this.totalHits = totalHits; + this.hits = hits; + } + + public ListIOCsActionResponse(StreamInput sin) throws IOException { + this(sin.readInt(), sin.readList(DetailedSTIX2IOCDto::new)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeLong(totalHits); + out.writeList(hits); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field(TOTAL_HITS_FIELD, totalHits) + .field(HITS_FIELD, hits) + .endObject(); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SADeleteTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SADeleteTIFSourceConfigAction.java new file mode 100644 index 000000000..bd27c7fa6 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SADeleteTIFSourceConfigAction.java @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionType; + +import static org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigAction.DELETE_TIF_SOURCE_CONFIG_ACTION_NAME; + +/** + * Delete TIF Source Config Action + */ +public class SADeleteTIFSourceConfigAction extends ActionType { + + public static final SADeleteTIFSourceConfigAction INSTANCE = new SADeleteTIFSourceConfigAction(); + public static final String NAME = DELETE_TIF_SOURCE_CONFIG_ACTION_NAME; + private SADeleteTIFSourceConfigAction() { + super(NAME, SADeleteTIFSourceConfigResponse::new); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SADeleteTIFSourceConfigRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SADeleteTIFSourceConfigRequest.java new file mode 100644 index 000000000..81955bf7d --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SADeleteTIFSourceConfigRequest.java @@ -0,0 +1,52 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.Locale; + +import static org.opensearch.action.ValidateActions.addValidationError; +import static org.opensearch.securityanalytics.threatIntel.common.Constants.THREAT_INTEL_SOURCE_CONFIG_ID; + +/** + * Delete threat intel feed source config request + */ +public class SADeleteTIFSourceConfigRequest extends ActionRequest { + private final String id; + public SADeleteTIFSourceConfigRequest(String id) { + super(); + this.id = id; + } + + public SADeleteTIFSourceConfigRequest(StreamInput sin) throws IOException { + this(sin.readString()); // id + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + } + + public String getId() { + return id; + } + + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (id == null || id.isEmpty()) { + validationException = addValidationError(String.format(Locale.getDefault(), "%s is missing", THREAT_INTEL_SOURCE_CONFIG_ID), validationException); + } + return validationException; + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SADeleteTIFSourceConfigResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SADeleteTIFSourceConfigResponse.java new file mode 100644 index 000000000..1fb37cb59 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SADeleteTIFSourceConfigResponse.java @@ -0,0 +1,58 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; + +import java.io.IOException; + +import static org.opensearch.securityanalytics.util.RestHandlerUtils._ID; +import static org.opensearch.securityanalytics.util.RestHandlerUtils._VERSION; + +public class SADeleteTIFSourceConfigResponse extends ActionResponse implements ToXContentObject { + private final String id; + private final RestStatus status; + + public SADeleteTIFSourceConfigResponse(String id, RestStatus status) { + super(); + this.id = id; + this.status = status; + } + + public SADeleteTIFSourceConfigResponse(StreamInput sin) throws IOException { + this( + sin.readString(), // id + sin.readEnum(RestStatus.class) // status + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(_ID, id); + return builder.endObject(); + } + + public String getId() { + return id; + } + + + public RestStatus getStatus() { + return status; + } + +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigAction.java new file mode 100644 index 000000000..f2a0099e7 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigAction.java @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionType; + +import static org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigAction.GET_TIF_SOURCE_CONFIG_ACTION_NAME; + +/** + * Get TIF Source Config Action + */ +public class SAGetTIFSourceConfigAction extends ActionType { + + public static final SAGetTIFSourceConfigAction INSTANCE = new SAGetTIFSourceConfigAction(); + public static final String NAME = GET_TIF_SOURCE_CONFIG_ACTION_NAME; + private SAGetTIFSourceConfigAction() { + super(NAME, SAGetTIFSourceConfigResponse::new); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigRequest.java new file mode 100644 index 000000000..9e067cabd --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigRequest.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.Locale; + +import static org.opensearch.action.ValidateActions.addValidationError; +import static org.opensearch.securityanalytics.threatIntel.common.Constants.THREAT_INTEL_SOURCE_CONFIG_ID; + +/** + * Get threat intel feed source config request + */ +public class SAGetTIFSourceConfigRequest extends ActionRequest { + private final String id; + private final Long version; + + public SAGetTIFSourceConfigRequest(String id, Long version) { + super(); + this.id = id; + this.version = version; + } + + public SAGetTIFSourceConfigRequest(StreamInput sin) throws IOException { + this(sin.readString(), // id + sin.readLong()); // version + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(version); + } + + public String getId() { + return id; + } + + public Long getVersion() { + return version; + } + + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (id == null || id.isEmpty()) { + validationException = addValidationError(String.format(Locale.getDefault(), "%s is missing", THREAT_INTEL_SOURCE_CONFIG_ID), validationException); + } + return validationException; + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigResponse.java new file mode 100644 index 000000000..7bebd8fb1 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigResponse.java @@ -0,0 +1,85 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; + +import java.io.IOException; + +import static org.opensearch.securityanalytics.util.RestHandlerUtils._ID; +import static org.opensearch.securityanalytics.util.RestHandlerUtils._VERSION; + +public class SAGetTIFSourceConfigResponse extends ActionResponse implements ToXContentObject { + private final String id; + + private final Long version; + + private final RestStatus status; + + private final SATIFSourceConfigDto saTifSourceConfigDto; + + + public SAGetTIFSourceConfigResponse(String id, Long version, RestStatus status, SATIFSourceConfigDto saTifSourceConfigDto) { + super(); + this.id = id; + this.version = version; + this.status = status; + this.saTifSourceConfigDto = saTifSourceConfigDto; + } + + public SAGetTIFSourceConfigResponse(StreamInput sin) throws IOException { + this( + sin.readString(), // id + sin.readLong(), // version + sin.readEnum(RestStatus.class), // status + sin.readBoolean()? SATIFSourceConfigDto.readFrom(sin) : null // SA tif config dto + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(version); + out.writeEnum(status); + if (saTifSourceConfigDto != null) { + out.writeBoolean((true)); + saTifSourceConfigDto.writeTo(out); + } else { + out.writeBoolean(false); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(_ID, id) + .field(_VERSION, version); + saTifSourceConfigDto.innerXcontent(builder); + return builder.endObject(); + } + + public String getId() { + return id; + } + + public Long getVersion() { + return version; + } + + public RestStatus getStatus() { + return status; + } + + public SATIFSourceConfigDto getSaTifSourceConfigDto() { + return saTifSourceConfigDto; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigAction.java new file mode 100644 index 000000000..1b4acd80e --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigAction.java @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionType; + +import static org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigAction.INDEX_TIF_SOURCE_CONFIG_ACTION_NAME; + +/** + * Threat intel tif job creation action + */ +public class SAIndexTIFSourceConfigAction extends ActionType { + + public static final SAIndexTIFSourceConfigAction INSTANCE = new SAIndexTIFSourceConfigAction(); + public static final String NAME = INDEX_TIF_SOURCE_CONFIG_ACTION_NAME; + private SAIndexTIFSourceConfigAction() { + super(NAME, SAIndexTIFSourceConfigResponse::new); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigRequest.java new file mode 100644 index 000000000..27fbc838b --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigRequest.java @@ -0,0 +1,85 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.rest.RestRequest; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigDtoValidator; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigRequest; + +import java.io.IOException; +import java.util.List; + +/** + * Threat intel feed config creation request + */ +public class SAIndexTIFSourceConfigRequest extends ActionRequest implements IndexTIFSourceConfigRequest { + private static final SourceConfigDtoValidator VALIDATOR = new SourceConfigDtoValidator(); + private String tifSourceConfigId; + private final RestRequest.Method method; + private SATIFSourceConfigDto saTifSourceConfigDto; + + public SAIndexTIFSourceConfigRequest(String tifSourceConfigId, + RestRequest.Method method, + SATIFSourceConfigDto saTifSourceConfigDto) { + super(); + this.tifSourceConfigId = tifSourceConfigId; + this.method = method; + this.saTifSourceConfigDto = saTifSourceConfigDto; + } + + public SAIndexTIFSourceConfigRequest(StreamInput sin) throws IOException { + this( + sin.readString(), // tif config id + sin.readEnum(RestRequest.Method.class), // method + SATIFSourceConfigDto.readFrom(sin) // SA tif config dto + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(tifSourceConfigId); + out.writeEnum(method); + saTifSourceConfigDto.writeTo(out); + } + + @Override + public String getTIFConfigId() { + return tifSourceConfigId; + } + + public void setTIFConfigId(String tifConfigId) { + this.tifSourceConfigId = tifConfigId; + } + + @Override + public SATIFSourceConfigDto getTIFConfigDto() { + return saTifSourceConfigDto; + } + + public void setTIFConfigDto(SATIFSourceConfigDto saTifSourceConfigDto) { + this.saTifSourceConfigDto = saTifSourceConfigDto; + } + + public RestRequest.Method getMethod() { + return method; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException errors = new ActionRequestValidationException(); + List errorMsgs = VALIDATOR.validateSourceConfigDto(saTifSourceConfigDto); + if (errorMsgs.isEmpty() == false) { + errorMsgs.forEach(errors::addValidationError); + } + return errors.validationErrors().isEmpty() ? null : errors; + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java new file mode 100644 index 000000000..209563f7c --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java @@ -0,0 +1,83 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigResponse; +import org.opensearch.securityanalytics.threatIntel.sacommons.TIFSourceConfigDto; + +import java.io.IOException; + +import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto.SOURCE_CONFIG_FIELD; +import static org.opensearch.securityanalytics.util.RestHandlerUtils._ID; +import static org.opensearch.securityanalytics.util.RestHandlerUtils._VERSION; + +public class SAIndexTIFSourceConfigResponse extends ActionResponse implements ToXContentObject, IndexTIFSourceConfigResponse { + private final String id; + private final Long version; + private final RestStatus status; + private final SATIFSourceConfigDto saTifSourceConfigDto; + + public SAIndexTIFSourceConfigResponse(String id, Long version, RestStatus status, SATIFSourceConfigDto saTifSourceConfigDto) { + super(); + this.id = id; + this.version = version; + this.status = status; + this.saTifSourceConfigDto = saTifSourceConfigDto; + } + + public SAIndexTIFSourceConfigResponse(StreamInput sin) throws IOException { + this( + sin.readString(), // tif config id + sin.readLong(), // version + sin.readEnum(RestStatus.class), // status + SATIFSourceConfigDto.readFrom(sin) // SA tif config dto + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(version); + out.writeEnum(status); + saTifSourceConfigDto.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(_ID, id) + .field(_VERSION, version); + saTifSourceConfigDto.innerXcontent(builder); + return builder.endObject(); + } + + @Override + public String getTIFConfigId() { + return id; + } + + @Override + public Long getVersion() { + return version; + } + + @Override + public TIFSourceConfigDto getTIFConfigDto() { + return saTifSourceConfigDto; + } + + public RestStatus getStatus() { + return status; + } + +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SARefreshTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SARefreshTIFSourceConfigAction.java new file mode 100644 index 000000000..cc84d946c --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SARefreshTIFSourceConfigAction.java @@ -0,0 +1,24 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionType; +import org.opensearch.action.support.master.AcknowledgedResponse; + +import static org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigAction.REFRESH_TIF_SOURCE_CONFIG_ACTION_NAME; + +/** + * Refresh TIF Source Config Action + */ +public class SARefreshTIFSourceConfigAction extends ActionType { + + public static final SARefreshTIFSourceConfigAction INSTANCE = new SARefreshTIFSourceConfigAction(); + + public static final String NAME = REFRESH_TIF_SOURCE_CONFIG_ACTION_NAME; + private SARefreshTIFSourceConfigAction() { + super(NAME, AcknowledgedResponse::new); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SARefreshTIFSourceConfigRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SARefreshTIFSourceConfigRequest.java new file mode 100644 index 000000000..abab39d3c --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SARefreshTIFSourceConfigRequest.java @@ -0,0 +1,53 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.Locale; + +import static org.opensearch.action.ValidateActions.addValidationError; +import static org.opensearch.securityanalytics.threatIntel.common.Constants.THREAT_INTEL_SOURCE_CONFIG_ID; + +/** + * Refresh threat intel feed source config request + */ +public class SARefreshTIFSourceConfigRequest extends ActionRequest { + private final String id; + + public SARefreshTIFSourceConfigRequest(String id) { + super(); + this.id = id; + } + + public SARefreshTIFSourceConfigRequest(StreamInput sin) throws IOException { + this(sin.readString()); // id + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + } + + public String getId() { + return id; + } + + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (id == null || id.isBlank()) { + validationException = addValidationError(String.format(Locale.getDefault(), "%s is missing", THREAT_INTEL_SOURCE_CONFIG_ID), validationException); + } + return validationException; + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SASearchTIFSourceConfigsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SASearchTIFSourceConfigsAction.java new file mode 100644 index 000000000..91284a5da --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SASearchTIFSourceConfigsAction.java @@ -0,0 +1,24 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionType; +import org.opensearch.action.search.SearchResponse; + +import static org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigAction.SEARCH_TIF_SOURCE_CONFIGS_ACTION_NAME; + +/** + * Search TIF Source Configs Action + */ +public class SASearchTIFSourceConfigsAction extends ActionType { + + public static final SASearchTIFSourceConfigsAction INSTANCE = new SASearchTIFSourceConfigsAction(); + + public static final String NAME = SEARCH_TIF_SOURCE_CONFIGS_ACTION_NAME; + private SASearchTIFSourceConfigsAction() { + super(NAME, SearchResponse::new); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SASearchTIFSourceConfigsRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SASearchTIFSourceConfigsRequest.java new file mode 100644 index 000000000..804cfc616 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SASearchTIFSourceConfigsRequest.java @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.search.builder.SearchSourceBuilder; + +import java.io.IOException; + +/** + * Search threat intel feed source config request + */ +public class SASearchTIFSourceConfigsRequest extends ActionRequest { + + // TODO: add pagination parameters + private final SearchSourceBuilder searchSourceBuilder; + + public SASearchTIFSourceConfigsRequest(SearchSourceBuilder searchSourceBuilder) { + super(); + this.searchSourceBuilder = searchSourceBuilder; + } + + public SASearchTIFSourceConfigsRequest(StreamInput sin) throws IOException { + searchSourceBuilder = new SearchSourceBuilder(sin); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + searchSourceBuilder.writeTo(out); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + public SearchSourceBuilder getSearchSourceBuilder() { + return searchSourceBuilder; + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/DeleteThreatIntelMonitorAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/DeleteThreatIntelMonitorAction.java new file mode 100644 index 000000000..5f22d21e4 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/DeleteThreatIntelMonitorAction.java @@ -0,0 +1,15 @@ +package org.opensearch.securityanalytics.threatIntel.action.monitor; + +import org.opensearch.action.ActionType; +import org.opensearch.commons.alerting.action.DeleteMonitorResponse; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorActions; + +public class DeleteThreatIntelMonitorAction extends ActionType { + + public static final DeleteThreatIntelMonitorAction INSTANCE = new DeleteThreatIntelMonitorAction(); + public static final String NAME = ThreatIntelMonitorActions.DELETE_THREAT_INTEL_MONITOR_ACTION_NAME; + + private DeleteThreatIntelMonitorAction() { + super(NAME, DeleteMonitorResponse::new); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/GetThreatIntelAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/GetThreatIntelAlertsAction.java new file mode 100644 index 000000000..16ba20543 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/GetThreatIntelAlertsAction.java @@ -0,0 +1,15 @@ +package org.opensearch.securityanalytics.threatIntel.action.monitor; + +import org.opensearch.action.ActionType; +import org.opensearch.securityanalytics.threatIntel.action.monitor.response.GetThreatIntelAlertsResponse; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorActions; + +public class GetThreatIntelAlertsAction extends ActionType { + + public static final GetThreatIntelAlertsAction INSTANCE = new GetThreatIntelAlertsAction(); + public static final String NAME = ThreatIntelMonitorActions.GET_THREAT_INTEL_ALERTS_ACTION_NAME; + + public GetThreatIntelAlertsAction() { + super(NAME, GetThreatIntelAlertsResponse::new); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/IndexThreatIntelMonitorAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/IndexThreatIntelMonitorAction.java new file mode 100644 index 000000000..e85ef09bf --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/IndexThreatIntelMonitorAction.java @@ -0,0 +1,17 @@ +package org.opensearch.securityanalytics.threatIntel.action.monitor; + +import org.opensearch.action.ActionType; +import org.opensearch.securityanalytics.threatIntel.action.monitor.response.IndexThreatIntelMonitorResponse; + +import static org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorActions.INDEX_THREAT_INTEL_MONITOR_ACTION_NAME; + + +public class IndexThreatIntelMonitorAction extends ActionType { + + public static final IndexThreatIntelMonitorAction INSTANCE = new IndexThreatIntelMonitorAction(); + public static final String NAME = INDEX_THREAT_INTEL_MONITOR_ACTION_NAME; + + private IndexThreatIntelMonitorAction() { + super(NAME, IndexThreatIntelMonitorResponse::new); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/SearchThreatIntelMonitorAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/SearchThreatIntelMonitorAction.java new file mode 100644 index 000000000..c57ff674e --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/SearchThreatIntelMonitorAction.java @@ -0,0 +1,15 @@ +package org.opensearch.securityanalytics.threatIntel.action.monitor; + +import org.opensearch.action.ActionType; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorActions; + +public class SearchThreatIntelMonitorAction extends ActionType { + + public static final SearchThreatIntelMonitorAction INSTANCE = new SearchThreatIntelMonitorAction(); + public static final String NAME = ThreatIntelMonitorActions.SEARCH_THREAT_INTEL_MONITOR_ACTION_NAME; + + private SearchThreatIntelMonitorAction() { + super(NAME, SearchResponse::new); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/UpdateThreatIntelAlertStatusAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/UpdateThreatIntelAlertStatusAction.java new file mode 100644 index 000000000..422eb052d --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/UpdateThreatIntelAlertStatusAction.java @@ -0,0 +1,15 @@ +package org.opensearch.securityanalytics.threatIntel.action.monitor; + +import org.opensearch.action.ActionType; +import org.opensearch.securityanalytics.threatIntel.action.monitor.response.UpdateThreatIntelAlertsStatusResponse; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorActions; + +public class UpdateThreatIntelAlertStatusAction extends ActionType { + + public static final UpdateThreatIntelAlertStatusAction INSTANCE = new UpdateThreatIntelAlertStatusAction(); + public static final String NAME = ThreatIntelMonitorActions.UPDATE_THREAT_INTEL_ALERT_STATUS_ACTION_NAME; + + public UpdateThreatIntelAlertStatusAction() { + super(NAME, UpdateThreatIntelAlertsStatusResponse::new); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/DeleteThreatIntelMonitorRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/DeleteThreatIntelMonitorRequest.java new file mode 100644 index 000000000..fcde1f299 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/DeleteThreatIntelMonitorRequest.java @@ -0,0 +1,37 @@ +package org.opensearch.securityanalytics.threatIntel.action.monitor.request; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; + +public class DeleteThreatIntelMonitorRequest extends ActionRequest { + + private String monitorId; + + public DeleteThreatIntelMonitorRequest(String monitorId) { + super(); + this.monitorId = monitorId; + } + + public DeleteThreatIntelMonitorRequest(StreamInput sin) throws IOException { + this(sin.readString()); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(monitorId); + } + + public String getMonitorId() { + return monitorId; + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/GetThreatIntelAlertsRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/GetThreatIntelAlertsRequest.java new file mode 100644 index 000000000..8d079fac5 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/GetThreatIntelAlertsRequest.java @@ -0,0 +1,106 @@ +package org.opensearch.securityanalytics.threatIntel.action.monitor.request; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.time.Instant; + +public class GetThreatIntelAlertsRequest extends ActionRequest { + + private final String monitorId; + private final Table table; + private final String severityLevel; + private final String alertState; + private final Instant startTime; + private final Instant endTime; + + public GetThreatIntelAlertsRequest( + String monitorId, + Table table, + String severityLevel, + String alertState, + Instant startTime, + Instant endTime + ) { + super(); + this.monitorId = monitorId; + this.table = table; + this.severityLevel = severityLevel; + this.alertState = alertState; + this.startTime = startTime; + this.endTime = endTime; + } + + public GetThreatIntelAlertsRequest( + Table table, + String severityLevel, + String alertState, + Instant startTime, + Instant endTime + ) { + super(); + this.monitorId = null; + this.table = table; + this.severityLevel = severityLevel; + this.alertState = alertState; + this.startTime = startTime; + this.endTime = endTime; + } + + public GetThreatIntelAlertsRequest(StreamInput sin) throws IOException { + this( + sin.readOptionalString(), + Table.readFrom(sin), + sin.readString(), + sin.readString(), + sin.readOptionalInstant(), + sin.readOptionalInstant() + ); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalString(monitorId); + table.writeTo(out); + out.writeString(severityLevel); + out.writeString(alertState); + out.writeOptionalInstant(startTime); + out.writeOptionalInstant(endTime); + } + + public String getmonitorId() { + return monitorId; + } + + public Table getTable() { + return table; + } + + public String getSeverityLevel() { + return severityLevel; + } + + public String getAlertState() { + return alertState; + } + + public Instant getStartTime() { + return startTime; + } + + public Instant getEndTime() { + return endTime; + } + + +} + diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/IndexThreatIntelMonitorRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/IndexThreatIntelMonitorRequest.java new file mode 100644 index 000000000..7f7205c5f --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/IndexThreatIntelMonitorRequest.java @@ -0,0 +1,59 @@ +package org.opensearch.securityanalytics.threatIntel.action.monitor.request; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.rest.RestRequest; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.IndexTIFSourceConfigRequestInterface; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto; + +import java.io.IOException; + +public class IndexThreatIntelMonitorRequest extends ActionRequest implements IndexTIFSourceConfigRequestInterface { + + public static final String THREAT_INTEL_MONITOR_ID = "threat_intel_monitor_id"; + + private final String id; + private final RestRequest.Method method; + private final ThreatIntelMonitorDto monitor; + + public IndexThreatIntelMonitorRequest(String id, RestRequest.Method method, ThreatIntelMonitorDto monitor) { + super(); + this.id = id; + this.method = method; + this.monitor = monitor; + } + + public IndexThreatIntelMonitorRequest(StreamInput sin) throws IOException { + this( + sin.readString(), + sin.readEnum(RestRequest.Method.class), // method + ThreatIntelMonitorDto.readFrom(sin) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeEnum(method); + monitor.writeTo(out); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + public String getId() { + return id; + } + + public RestRequest.Method getMethod() { + return method; + } + + public ThreatIntelMonitorDto getMonitor() { + return monitor; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/SearchThreatIntelMonitorRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/SearchThreatIntelMonitorRequest.java new file mode 100644 index 000000000..8c80209b2 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/SearchThreatIntelMonitorRequest.java @@ -0,0 +1,36 @@ +package org.opensearch.securityanalytics.threatIntel.action.monitor.request; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; + +public class SearchThreatIntelMonitorRequest extends ActionRequest { + private SearchRequest searchRequest; + + public SearchThreatIntelMonitorRequest(SearchRequest searchRequest) { + super(); + this.searchRequest = searchRequest; + } + + public SearchThreatIntelMonitorRequest(StreamInput sin) throws IOException { + searchRequest = new SearchRequest(sin); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + searchRequest.writeTo(out); + } + + public SearchRequest searchRequest() { + return this.searchRequest; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/UpdateThreatIntelAlertStatusRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/UpdateThreatIntelAlertStatusRequest.java new file mode 100644 index 000000000..4388d98a3 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/UpdateThreatIntelAlertStatusRequest.java @@ -0,0 +1,75 @@ +package org.opensearch.securityanalytics.threatIntel.action.monitor.request; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.commons.alerting.model.Alert; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.List; + +public class UpdateThreatIntelAlertStatusRequest extends ActionRequest { + public static final String ALERT_IDS_FIELD = "alert_ids"; + public static final String STATE_FIELD = "state"; + private final List alertIds; + private final Alert.State state; + private final String monitorId; + + public UpdateThreatIntelAlertStatusRequest(StreamInput sin) throws IOException { + alertIds = sin.readStringList(); + state = sin.readEnum(Alert.State.class); + monitorId = sin.readOptionalString(); + } + + public UpdateThreatIntelAlertStatusRequest(List alertIds, Alert.State state) { + this.alertIds = alertIds; + this.state = state; + monitorId = null; + } + + public UpdateThreatIntelAlertStatusRequest(List alertIds, String monitorId, Alert.State state) { + this.alertIds = alertIds; + this.state = state; + this.monitorId = monitorId; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeStringCollection(alertIds); + out.writeEnum(state); + out.writeOptionalString(monitorId); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException actionRequestValidationException = null; + + if (state == null) { + actionRequestValidationException = new ActionRequestValidationException(); + actionRequestValidationException.addValidationError("State cannot be null"); + } + if (alertIds == null || alertIds.isEmpty()) { + actionRequestValidationException = new ActionRequestValidationException(); + actionRequestValidationException.addValidationError("At least one alert id is required"); + } + if (false == (state.equals(Alert.State.ACKNOWLEDGED) || state.equals(Alert.State.COMPLETED))) { + actionRequestValidationException = new ActionRequestValidationException(); + actionRequestValidationException.addValidationError(String.format("%s is not a supported state for alert status update." + + " Only COMPLETED and ACKNOWLEDGED states allowed", state.toString())); + } + return actionRequestValidationException; + } + + public List getAlertIds() { + return alertIds; + } + + public Alert.State getState() { + return state; + } + + public String getMonitorId() { + return monitorId; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/response/GetThreatIntelAlertsResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/response/GetThreatIntelAlertsResponse.java new file mode 100644 index 000000000..1e3895dab --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/response/GetThreatIntelAlertsResponse.java @@ -0,0 +1,57 @@ +package org.opensearch.securityanalytics.threatIntel.action.monitor.response; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelAlertDto; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class GetThreatIntelAlertsResponse extends ActionResponse implements ToXContentObject { + + private static final String ALERTS_FIELD = "alerts"; + private static final String TOTAL_ALERTS_FIELD = "total_alerts"; + + private List alerts; + private Integer totalAlerts; + + public GetThreatIntelAlertsResponse(List alerts, Integer totalAlerts) { + super(); + this.alerts = alerts; + this.totalAlerts = totalAlerts; + } + + public GetThreatIntelAlertsResponse(StreamInput sin) throws IOException { + this( + Collections.unmodifiableList(sin.readList(ThreatIntelAlertDto::new)), + sin.readInt() + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeCollection(this.alerts); + out.writeInt(this.totalAlerts); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(ALERTS_FIELD, alerts) + .field(TOTAL_ALERTS_FIELD, totalAlerts); + return builder.endObject(); + } + + public List getAlerts() { + return this.alerts; + } + + public Integer getTotalAlerts() { + return this.totalAlerts; + } +} + diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/response/IndexThreatIntelMonitorResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/response/IndexThreatIntelMonitorResponse.java new file mode 100644 index 000000000..332198f4c --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/response/IndexThreatIntelMonitorResponse.java @@ -0,0 +1,89 @@ +package org.opensearch.securityanalytics.threatIntel.action.monitor.response; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.IndexIocScanMonitorResponseInterface; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto; + +import java.io.IOException; + +/** + * Response object resturned for request that indexes ioc scan monitor + */ +public class IndexThreatIntelMonitorResponse extends ActionResponse implements ToXContentObject, IndexIocScanMonitorResponseInterface { + private static final String ID = "id"; + private static final String NAME = "name"; + private static final String SEQ_NO = "seq_no"; + private static final String PRIMARY_TERM = "primary_term"; + private static final String MONITOR = "monitor"; + + private final String id; + private final long version; + private final long seqNo; + private final long primaryTerm; + private final ThreatIntelMonitorDto iocScanMonitor; + + public IndexThreatIntelMonitorResponse(String id, long version, long seqNo, long primaryTerm, ThreatIntelMonitorDto monitor) { + this.id = id; + this.version = version; + this.seqNo = seqNo; + this.primaryTerm = primaryTerm; + this.iocScanMonitor = monitor; + } + + public IndexThreatIntelMonitorResponse(StreamInput sin) throws IOException { + this( + sin.readString(), + sin.readLong(), // version + sin.readLong(), // seqNo + sin.readLong(), // primaryTerm + ThreatIntelMonitorDto.readFrom(sin) // monitor + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(version); + out.writeLong(seqNo); + out.writeLong(primaryTerm); + iocScanMonitor.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + return builder.startObject() + .field(ID, id) + .field(NAME, version) + .field(SEQ_NO, seqNo) + .field(PRIMARY_TERM, primaryTerm) + .field(MONITOR, iocScanMonitor) + .endObject(); + } + + @Override + public String getId() { + return id; + } + + public Long getVersion() { + return version; + } + + public long getSeqNo() { + return seqNo; + } + + public long getPrimaryTerm() { + return primaryTerm; + } + + @Override + public ThreatIntelMonitorDto getIocScanMonitor() { + return iocScanMonitor; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/response/UpdateThreatIntelAlertsStatusResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/response/UpdateThreatIntelAlertsStatusResponse.java new file mode 100644 index 000000000..422df8eb2 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/response/UpdateThreatIntelAlertsStatusResponse.java @@ -0,0 +1,45 @@ +package org.opensearch.securityanalytics.threatIntel.action.monitor.response; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelAlertDto; + +import java.io.IOException; +import java.util.List; + +public class UpdateThreatIntelAlertsStatusResponse extends ActionResponse implements ToXContentObject { + public static final String UPDATED_ALERTS = "updated_alerts"; + public static final String FAILURE_MESSAGES_FIELD = "failure_messages"; + private final List updatedAlerts; + private final List failureMessages; + + public UpdateThreatIntelAlertsStatusResponse( + List updatedAlerts, + List failureMessages + ) { + this.updatedAlerts = updatedAlerts; + this.failureMessages = failureMessages; + } + + public UpdateThreatIntelAlertsStatusResponse(StreamInput sin) throws IOException { + updatedAlerts = sin.readList(ThreatIntelAlertDto::new); + failureMessages = sin.readStringList(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeCollection(updatedAlerts); + out.writeStringCollection(failureMessages); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field(UPDATED_ALERTS, updatedAlerts.toArray(new ThreatIntelAlertDto[0])) + .field(FAILURE_MESSAGES_FIELD, failureMessages) + .endObject(); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/Constants.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/Constants.java index 808c0a3da..c69c16294 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/Constants.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/Constants.java @@ -10,4 +10,5 @@ public class Constants { public static final String USER_AGENT_KEY = "User-Agent"; public static final String USER_AGENT_VALUE = String.format(Locale.ROOT, "OpenSearch/%s vanilla", Version.CURRENT.toString()); + public static final String THREAT_INTEL_SOURCE_CONFIG_ID = "threat_intel_source_config_id"; } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/RefreshType.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/RefreshType.java new file mode 100644 index 000000000..0ac915781 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/RefreshType.java @@ -0,0 +1,15 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.common; + +/** + * Refresh Types: Full + * TODO: Add other refresh types such as the delta + */ +public enum RefreshType { + + FULL +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java new file mode 100644 index 000000000..4a5ab1446 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java @@ -0,0 +1,100 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.common; + +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource; +import org.opensearch.securityanalytics.threatIntel.model.S3Source; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; + +/** + * Source config dto validator + */ +public class SourceConfigDtoValidator { + public List validateSourceConfigDto(SATIFSourceConfigDto sourceConfigDto) { + List errorMsgs = new ArrayList<>(); + + String nameRegex = "^[a-zA-Z0-9 _-]{1,128}$"; + Pattern namePattern = Pattern.compile(nameRegex); + + int MAX_RULE_DESCRIPTION_LENGTH = 65535; + String descriptionRegex = "^.{0," + MAX_RULE_DESCRIPTION_LENGTH + "}$"; + Pattern descriptionPattern = Pattern.compile(descriptionRegex); + + if (sourceConfigDto.getName() == null || sourceConfigDto.getName().isEmpty()) { + errorMsgs.add("Name must not be empty"); + } else if (sourceConfigDto.getName() != null && namePattern.matcher(sourceConfigDto.getName()).matches() == false) { + errorMsgs.add("Name must be less than 128 characters and only consist of upper and lowercase letters, numbers 0-9, hyphens, spaces, and underscores"); + } + + if (sourceConfigDto.getFormat() == null || sourceConfigDto.getFormat().isEmpty()) { + errorMsgs.add("Format must not be empty"); + } else if (sourceConfigDto.getFormat() != null && sourceConfigDto.getFormat().length() > 50) { + errorMsgs.add("Format must be 50 characters or less"); + } + + if (sourceConfigDto.getDescription() != null && descriptionPattern.matcher(sourceConfigDto.getDescription()).matches() == false) { + errorMsgs.add("Description must be " + MAX_RULE_DESCRIPTION_LENGTH + " characters or less"); + } + + if (sourceConfigDto.getSource() == null) { + errorMsgs.add("Source must not be empty"); + } + + if (sourceConfigDto.getIocTypes() == null || sourceConfigDto.getIocTypes().isEmpty()) { + errorMsgs.add("Must specify at least one IOC type"); + } else { + for (String s: sourceConfigDto.getIocTypes()) { + if (!IOCType.supportedType(s)) { + errorMsgs.add("Invalid IOC type: " + s); + } + } + } + + if (sourceConfigDto.getType() == null) { + errorMsgs.add("Type must not be empty"); + } else { + switch (sourceConfigDto.getType()) { + case IOC_UPLOAD: + if (sourceConfigDto.isEnabled()) { + errorMsgs.add("Job Scheduler cannot be enabled for IOC_UPLOAD type"); + } + if (sourceConfigDto.getSchedule() != null) { + errorMsgs.add("Cannot pass in schedule for IOC_UPLOAD type"); + } + if (sourceConfigDto.getSource() != null && sourceConfigDto.getSource() instanceof IocUploadSource == false) { + errorMsgs.add("Source must be IOC_UPLOAD type"); + } + if (sourceConfigDto.getSource() instanceof IocUploadSource && ((IocUploadSource) sourceConfigDto.getSource()).getIocs() == null) { + errorMsgs.add("Ioc list must include at least one ioc"); + } + break; + case S3_CUSTOM: + if (sourceConfigDto.getSchedule() == null) { + errorMsgs.add("Must pass in schedule for S3_CUSTOM type"); + } + if (sourceConfigDto.getSource() != null && sourceConfigDto.getSource() instanceof S3Source == false) { + errorMsgs.add("Source must be S3_CUSTOM type"); + } + break; + case URL_DOWNLOAD: + if (sourceConfigDto.getSchedule() == null) { + errorMsgs.add("Must pass in schedule for URL_DOWNLOAD source type"); + } + if (sourceConfigDto.getSource() != null && sourceConfigDto.getSource() instanceof UrlDownloadSource == false) { + errorMsgs.add("Source must be URL_DOWNLOAD source type"); + } + break; + } + } + return errorMsgs; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigType.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigType.java new file mode 100644 index 000000000..8efa5cfa5 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigType.java @@ -0,0 +1,26 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.common; + +/** + * Types of feeds threat intel can support + */ +public enum SourceConfigType { + S3_CUSTOM, + IOC_UPLOAD, + URL_DOWNLOAD + +// LICENSED, +// +// OPEN_SOURCED, +// +// INTERNAL, +// +// DEFAULT_OPEN_SOURCED, +// +// EXTERNAL_LICENSED, + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFJobState.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFJobState.java index 22ffee3e9..db72ac757 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFJobState.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFJobState.java @@ -33,5 +33,15 @@ public enum TIFJobState { /** * tif job is being deleted */ - DELETING + DELETING, + + /** + * tif associated iocs are being refreshed + */ + REFRESHING, + + /** + * tif refresh job failed + */ + REFRESH_FAILED } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java index 98abf040a..eb467c55e 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java @@ -46,7 +46,7 @@ public TIFLockService(final ClusterService clusterService, final Client client) } /** - * Synchronous method of #acquireLock + * Event-driven method of #acquireLock * * @param tifJobName tifJobName to acquire lock on * @param lockDurationSeconds the lock duration in seconds @@ -73,10 +73,13 @@ public void onFailure(final Exception e) { * * @param lockModel the lock model */ - public void releaseLock(final LockModel lockModel) { + public void releaseLock(final LockModel lockModel, final ActionListener listener) { lockService.release( lockModel, - ActionListener.wrap(released -> {}, exception -> log.error("Failed to release the lock", exception)) + ActionListener.wrap(listener::onResponse, exception -> { + log.error("Failed to release the lock", exception); + listener.onFailure(exception); + }) ); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/feedMetadata/BuiltInTIFMetadataLoader.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/feedMetadata/BuiltInTIFMetadataLoader.java index 6b84e9fe9..2b5856999 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/feedMetadata/BuiltInTIFMetadataLoader.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/feedMetadata/BuiltInTIFMetadataLoader.java @@ -10,7 +10,7 @@ import org.opensearch.common.settings.SettingsException; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; import org.opensearch.securityanalytics.util.FileUtils; import java.io.IOException; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/BaseEntityCrudService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/BaseEntityCrudService.java new file mode 100644 index 000000000..f435db5ce --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/BaseEntityCrudService.java @@ -0,0 +1,267 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.dao; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.ResourceAlreadyExistsException; +import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.rest.action.admin.indices.AliasesNotFoundException; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.model.threatintel.BaseEntity; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.transport.RemoteTransportException; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; +import static org.opensearch.securityanalytics.util.DetectorUtils.getEmptySearchResponse; + +/** + * Provides generic CRUD implementations for entity that is stored in system index. Provides generic implementation + * of system index too. + */ +public abstract class BaseEntityCrudService { + // todo rollover + private static final Logger log = LogManager.getLogger(BaseEntityCrudService.class); + private final Client client; + private final ClusterService clusterService; + private final NamedXContentRegistry xContentRegistry; + + public BaseEntityCrudService(Client client, ClusterService clusterService, NamedXContentRegistry xContentRegistry) { + this.client = client; + this.clusterService = clusterService; + this.xContentRegistry = xContentRegistry; + } + + + public void bulkIndexEntities(List newEntityList, List updatedEntityList, + ActionListener actionListener) { + try { + Integer batchSize = this.clusterService.getClusterSettings().get(SecurityAnalyticsSettings.BATCH_SIZE); + createIndexIfNotExists(ActionListener.wrap( + r -> { + List bulkRequestList = new ArrayList<>(); + BulkRequest bulkRequest = new BulkRequest(getEntityAliasName()); + for (int i = 0; i < newEntityList.size(); i++) { + Entity entity = newEntityList.get(i); + try { + IndexRequest indexRequest = new IndexRequest(getEntityAliasName()) + .id(entity.getId()) + .source(entity.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .opType(DocWriteRequest.OpType.CREATE); + bulkRequest.add(indexRequest); + if ( + bulkRequest.requests().size() == batchSize + && i != newEntityList.size() - 1 // final bulk request will be added outside for loop with refresh policy none + ) { + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE); + bulkRequestList.add(bulkRequest); + bulkRequest = new BulkRequest(); + } + } catch (IOException e) { + log.error(String.format("Failed to create index request for %s moving on to next", getEntityName()), e); + } + } + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + bulkRequestList.add(bulkRequest); + for (int i = 0; i < updatedEntityList.size(); i++) { + Entity entity = updatedEntityList.get(i); + try { + IndexRequest indexRequest = new IndexRequest(getEntityAliasName()) + .id(entity.getId()) + .source(entity.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .opType(DocWriteRequest.OpType.INDEX); + bulkRequest.add(indexRequest); + if ( + bulkRequest.requests().size() == batchSize + && i != updatedEntityList.size() - 1 // final bulk request will be added outside for loop with refresh policy none + ) { + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE); + bulkRequestList.add(bulkRequest); + bulkRequest = new BulkRequest(); + } + } catch (IOException e) { + log.error(String.format("Failed to create index request for %s moving on to next", getEntityName()), e); + } + } + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + bulkRequestList.add(bulkRequest); + GroupedActionListener groupedListener = new GroupedActionListener<>(ActionListener.wrap(bulkResponses -> { + int idx = 0; + for (BulkResponse response : bulkResponses) { + BulkRequest request = bulkRequestList.get(idx); + if (response.hasFailures()) { + log.error("Failed to bulk index {} {}s. Failure: {}", request.requests().size(), getEntityName(), response.buildFailureMessage()); + } + } + actionListener.onResponse(null); + }, e1 -> { + log.error("Failed to bulk index " + getEntityName(), e1); + actionListener.onFailure(e1); + }), bulkRequestList.size()); + + for (BulkRequest req : bulkRequestList) { + try { + client.bulk(req, groupedListener); + } catch (Exception e) { + log.error( + () -> new ParameterizedMessage("Failed to bulk save {} {}.", req.batchSize(), getEntityName()), + e); + groupedListener.onFailure(e); + } + } + }, e -> { + log.error(() -> new ParameterizedMessage("Failed to create System Index {}", getEntityAliasName()), e); + actionListener.onFailure(e); + })); + + + } catch (Exception e) { + log.error("Exception saving the threat intel source config in index", e); + actionListener.onFailure(e); + } + } + + public void bulkIndexEntities(List entityList, + ActionListener actionListener) { + try { + Integer batchSize = this.clusterService.getClusterSettings().get(SecurityAnalyticsSettings.BATCH_SIZE); + createIndexIfNotExists(ActionListener.wrap( + r -> { + List bulkRequestList = new ArrayList<>(); + BulkRequest bulkRequest = new BulkRequest(getEntityAliasName()); + for (int i = 0; i < entityList.size(); i++) { + Entity entity = entityList.get(i); + try { + IndexRequest indexRequest = new IndexRequest(getEntityAliasName()) + .id(entity.getId()) + .source(entity.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .opType(DocWriteRequest.OpType.CREATE); + bulkRequest.add(indexRequest); + if ( + bulkRequest.requests().size() == batchSize + && i != entityList.size() - 1 // final bulk request will be added outside for loop with refresh policy none + ) { + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE); + bulkRequestList.add(bulkRequest); + bulkRequest = new BulkRequest(); + } + } catch (IOException e) { + log.error(String.format("Failed to create index request for %s moving on to next", getEntityName()), e); + } + } + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + bulkRequestList.add(bulkRequest); + GroupedActionListener groupedListener = new GroupedActionListener<>(ActionListener.wrap(bulkResponses -> { + int idx = 0; + for (BulkResponse response : bulkResponses) { + BulkRequest request = bulkRequestList.get(idx); + if (response.hasFailures()) { + log.error("Failed to bulk index {} {}s. Failure: {}", request.batchSize(), getEntityName(), response.buildFailureMessage()); + } + } + actionListener.onResponse(null); + }, actionListener::onFailure), bulkRequestList.size()); + for (BulkRequest req : bulkRequestList) { + try { + client.bulk(req, groupedListener); //todo why stash context here? + } catch (Exception e) { + log.error( + () -> new ParameterizedMessage("Failed to bulk save {} {}.", req.batchSize(), getEntityName()), + e); + } + } + }, e -> { + log.error(() -> new ParameterizedMessage("Failed to create System Index {}", getEntityIndexPattern()), e); + actionListener.onFailure(e); + })); + + + } catch (Exception e) { + log.error("Exception saving the threat intel source config in index", e); + actionListener.onFailure(e); + } + } + + public void search(SearchSourceBuilder searchSourceBuilder, final ActionListener listener) { + SearchRequest searchRequest = new SearchRequest() + .source(searchSourceBuilder) + .indices(getEntityAliasName()); + client.search(searchRequest, ActionListener.wrap( + listener::onResponse, + e -> { + if (e instanceof IndexNotFoundException || e instanceof AliasesNotFoundException) { + listener.onResponse(getEmptySearchResponse()); + return; + } + log.error( + () -> new ParameterizedMessage("Failed to search {}s from index {}.", getEntityName(), getEntityAliasName()), + e); + listener.onFailure(e); + } + )); + } + + public void createIndexIfNotExists(final ActionListener listener) { + try { + if (clusterService.state().metadata().hasAlias(getEntityAliasName())) { + listener.onResponse(null); + return; + } + final CreateIndexRequest createIndexRequest = new CreateIndexRequest(getEntityIndexPattern()).mapping(getEntityIndexMapping()) + .settings(getIndexSettings()); + client.admin().indices().create(createIndexRequest, ActionListener.wrap( + r -> { + log.debug("{} index created", getEntityName()); + listener.onResponse(null); + }, e -> { + if (e instanceof ResourceAlreadyExistsException || (e instanceof RemoteTransportException && e.getCause() instanceof ResourceAlreadyExistsException)) { + log.debug("index {} already exist", getEntityIndexMapping()); + listener.onResponse(null); + return; + } + log.error(String.format("Failed to create security analytics threat intel %s index", getEntityName()), e); + listener.onFailure(e); + } + )); + } catch (Exception e) { + log.error(String.format("Failure in creating %s index", getEntityName()), e); + listener.onFailure(e); + } + } + + protected abstract String getEntityIndexMapping(); + + public abstract String getEntityName(); + + protected Settings.Builder getIndexSettings() { + return Settings.builder().put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas); + } + + public abstract String getEntityAliasName(); + + public abstract String getEntityIndexPattern(); + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingService.java new file mode 100644 index 000000000..eaf94bdbf --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingService.java @@ -0,0 +1,74 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.dao; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.stream.Collectors; + +/** + * Data layer to perform CRUD operations for threat intel ioc finding : store in system index. + */ +public class IocFindingService extends BaseEntityCrudService { + + public static final String IOC_FINDING_ALIAS_NAME = ".opensearch-sap-ioc-findings"; + + public static final String IOC_FINDING_INDEX_PATTERN = "<.opensearch-sap-ioc-findings-history-{now/d}-1>"; + + public static final String IOC_FINDING_INDEX_PATTERN_REGEXP = ".opensearch-sap-ioc-findings*"; + + private static final Logger log = LogManager.getLogger(IocFindingService.class); + private final Client client; + private final ClusterService clusterService; + + private final NamedXContentRegistry xContentRegistry; + + public IocFindingService(final Client client, final ClusterService clusterService, final NamedXContentRegistry xContentRegistry) { + super(client, clusterService, xContentRegistry); + this.client = client; + this.clusterService = clusterService; + this.xContentRegistry = xContentRegistry; + } + + @Override + public String getEntityIndexMapping() { + return getIndexMapping(); + } + + public static String getIndexMapping() { + try { + try (InputStream is = IocFindingService.class.getResourceAsStream("/mappings/ioc_finding_mapping.json")) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + return reader.lines().map(String::trim).collect(Collectors.joining()); + } + } + } catch (IOException e) { + log.error("Failed to get the threat intel ioc finding index mapping", e); + throw new SecurityAnalyticsException("Failed to get the threat intel ioc finding index mapping", RestStatus.INTERNAL_SERVER_ERROR, e); + } + } + @Override + public String getEntityAliasName() { + return IOC_FINDING_ALIAS_NAME; + } + + @Override + public String getEntityIndexPattern() { + return IOC_FINDING_INDEX_PATTERN; + } + + @Override + public String getEntityName() { + return "ioc_finding"; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/ThreatIntelAlertService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/ThreatIntelAlertService.java new file mode 100644 index 000000000..987203cda --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/ThreatIntelAlertService.java @@ -0,0 +1,66 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.dao; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.securityanalytics.model.threatintel.ThreatIntelAlert; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.stream.Collectors; + +public class ThreatIntelAlertService extends BaseEntityCrudService { + + public static final String THREAT_INTEL_ALERT_ALIAS_NAME = ".opensearch-sap-threat-intel-alerts"; + + public static final String THREAT_INTEL_ALERT_INDEX_PATTERN = "<.opensearch-sap-threat-intel-alerts-history-{now/d}-1>"; + + public static final String THREAT_INTEL_ALERT_INDEX_PATTERN_REGEXP = ".opensearch-sap-threat-intel-alerts*"; + + private static final Logger log = LogManager.getLogger(ThreatIntelAlertService.class); + + public ThreatIntelAlertService(Client client, ClusterService clusterService, NamedXContentRegistry xContentRegistry) { + super(client, clusterService, xContentRegistry); + } + + @Override + protected String getEntityIndexMapping() { + return getIndexMapping(); + } + + public static String getIndexMapping() { + try { + try (InputStream is = IocFindingService.class.getResourceAsStream("/mappings/threat_intel_alert_mapping.json")) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + return reader.lines().map(String::trim).collect(Collectors.joining()); + } + } + } catch (IOException e) { + log.error("Failed to get the threat intel alert index mapping", e); + throw new SecurityAnalyticsException("Failed to get the threat intel alert index mapping", RestStatus.INTERNAL_SERVER_ERROR, e); + } + } + + @Override + public String getEntityName() { + return "threat_intel_alert"; + } + + @Override + public String getEntityAliasName() { + return THREAT_INTEL_ALERT_ALIAS_NAME; + } + + @Override + public String getEntityIndexPattern() { + return THREAT_INTEL_ALERT_INDEX_PATTERN; + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/IocScanContext.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/IocScanContext.java new file mode 100644 index 000000000..4a062e718 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/IocScanContext.java @@ -0,0 +1,63 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.dto; + +import org.opensearch.commons.alerting.model.Monitor; +import org.opensearch.commons.alerting.model.MonitorMetadata; +import org.opensearch.securityanalytics.threatIntel.model.monitor.ThreatIntelInput; + +import java.util.List; +import java.util.Map; + +public class IocScanContext { + private final Monitor monitor; + private final MonitorMetadata monitorMetadata; + private final boolean dryRun; + private final List data; + private final ThreatIntelInput threatIntelInput; // deserialize threat intel input + private final List indices; // user's log data indices + private final Map> iocTypeToIndices; + private final Map> concreteIndexToMonitorInputIndicesMap; + + public IocScanContext(Monitor monitor, MonitorMetadata monitorMetadata, boolean dryRun, List data, ThreatIntelInput threatIntelInput, List indices, Map> iocTypeToIndices, Map> concreteIndexToMonitorInputIndicesMap) { + this.monitor = monitor; + this.monitorMetadata = monitorMetadata; + this.dryRun = dryRun; + this.data = data; + this.threatIntelInput = threatIntelInput; + this.indices = indices; + this.iocTypeToIndices = iocTypeToIndices; + this.concreteIndexToMonitorInputIndicesMap = concreteIndexToMonitorInputIndicesMap; + } + + public Monitor getMonitor() { + return monitor; + } + + public boolean isDryRun() { + return dryRun; + } + + public List getData() { + return data; + } + + public MonitorMetadata getMonitorMetadata() { + return monitorMetadata; + } + + public ThreatIntelInput getThreatIntelInput() { + return threatIntelInput; + } + + public List getIndices() { + return indices; + } + + public Map> getConcreteIndexToMonitorInputIndicesMap() { + return concreteIndexToMonitorInputIndicesMap; + } + + public Map> getIocTypeToIndices() { + return iocTypeToIndices; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/PerIocTypeScanInputDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/PerIocTypeScanInputDto.java new file mode 100644 index 000000000..8d0756c18 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/PerIocTypeScanInputDto.java @@ -0,0 +1,98 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.dto; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * DTO that contains information about an Ioc type, the indices storing iocs of that ioc type and + * list of fields in each index that contain values of the given ioc type like Ip addresss contain fields. + * If indices is empty we scan the feed config and get the list of indices + */ +public class PerIocTypeScanInputDto implements Writeable, ToXContentObject { + + private static final String IOC_TYPE = "ioc_type"; + private static final String INDEX_TO_FIELDS_MAP = "index_to_fields_map"; + private final String iocType; + private final Map> indexToFieldsMap; + + public PerIocTypeScanInputDto(String iocType, Map> indexToFieldsMap) { + this.iocType = iocType; + this.indexToFieldsMap = indexToFieldsMap == null ? Collections.emptyMap() : indexToFieldsMap; + } + + public PerIocTypeScanInputDto(StreamInput sin) throws IOException { + this( + sin.readString(), + sin.readMapOfLists(StreamInput::readString, StreamInput::readString) + ); + } + + public String getIocType() { + return iocType; + } + + public Map> getIndexToFieldsMap() { + return indexToFieldsMap; + } + + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(iocType); + out.writeMapOfLists(indexToFieldsMap, StreamOutput::writeString, StreamOutput::writeString); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field(IOC_TYPE, iocType) + .field(INDEX_TO_FIELDS_MAP, indexToFieldsMap) + .endObject(); + } + + public static PerIocTypeScanInputDto parse(XContentParser xcp) throws IOException { + String iocType = ""; + Map> indexToFieldsMap = new HashMap<>(); + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case IOC_TYPE: + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) iocType = xcp.text(); + break; + case INDEX_TO_FIELDS_MAP: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + indexToFieldsMap = null; + } else { + indexToFieldsMap = xcp.map(HashMap::new, p -> { + List fields = new ArrayList<>(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) fields.add(xcp.text()); + } + return fields; + }); + } + break; + default: + xcp.skipChildren(); + } + } + return new PerIocTypeScanInputDto(iocType, indexToFieldsMap); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java new file mode 100644 index 000000000..f60af7afd --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java @@ -0,0 +1,247 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.service; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.commons.alerting.model.Monitor; +import org.opensearch.securityanalytics.model.STIX2IOC; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; +import org.opensearch.securityanalytics.model.threatintel.IocWithFeeds; +import org.opensearch.securityanalytics.model.threatintel.ThreatIntelAlert; +import org.opensearch.securityanalytics.threatIntel.iocscan.dto.IocScanContext; +import org.opensearch.securityanalytics.threatIntel.model.monitor.PerIocTypeScanInput; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.BiConsumer; + + +public abstract class IoCScanService implements IoCScanServiceInterface { + private static final Logger log = LogManager.getLogger(IoCScanService.class); + + @Override + public void scanIoCs(IocScanContext iocScanContext, + BiConsumer scanCallback + ) { + try { + List data = iocScanContext.getData(); + if (data.isEmpty()) { + scanCallback.accept(Collections.emptyList(), null); + return; + } + Monitor monitor = iocScanContext.getMonitor(); + + long startTime = System.currentTimeMillis(); + IocLookupDtos iocLookupDtos = extractIocsPerType(data, iocScanContext); + if (iocLookupDtos.getIocsPerIocTypeMap().isEmpty()) { + log.error("Threat intel monitor {}: Unexpected scenario that non-zero number of docs are fetched from indices containing iocs but iocs-per-type map constructed is empty", + iocScanContext.getMonitor().getId() + ); + scanCallback.accept(Collections.emptyList(), null); + return; + } + BiConsumer, Exception> iocScanResultConsumer = (List maliciousIocs, Exception e) -> { + long scanEndTime = System.currentTimeMillis(); + long timeTaken = scanEndTime - startTime; + log.debug("Threat intel monitor {}: scan time taken is {}", monitor.getId(), timeTaken); + if (e == null) { + createIocFindings(maliciousIocs, iocLookupDtos.iocValueToDocIdMap, iocScanContext, + (iocFindings, e1) -> { + if (e1 != null) { + log.error( + () -> new ParameterizedMessage("Threat intel monitor {}: Failed to create ioc findings", + iocScanContext.getMonitor().getId(), data.size()), + e1); + scanCallback.accept(data, e1); + } else { + BiConsumer, Exception> triggerResultConsumer = (alerts, e2) -> { + if (e2 != null) { + log.error( + () -> new ParameterizedMessage("Threat intel monitor {}: Failed to execute threat intel triggers/ ", + iocScanContext.getMonitor().getId(), data.size()), + e2); + // if findings are generated successfully but alerts/notifications fail we mark execution as succeeded, so that duplicate findings are not created + scanCallback.accept(data, null); + } else { + scanCallback.accept(data, null); + } + }; + executeTriggers(maliciousIocs, iocFindings, iocScanContext, data, iocLookupDtos, + triggerResultConsumer); + + } + + } + ); + } else { + log.error( + () -> new ParameterizedMessage("Threat intel monitor {}: Failed to run scan for {} docs", + iocScanContext.getMonitor().getId(), data.size()), + e); + scanCallback.accept(null, e); + + } + }; + matchAgainstThreatIntelAndReturnMaliciousIocs( + iocLookupDtos.getIocsPerIocTypeMap(), monitor, iocScanResultConsumer, iocScanContext.getIocTypeToIndices()); + } catch (Exception e) { + log.error( + () -> new ParameterizedMessage("Threat intel monitor {}: Unexpected failure in running scan for {} docs", + iocScanContext.getMonitor().getId(), iocScanContext.getData().size()), + e); + scanCallback.accept(null, e); + } + } + + + abstract void executeTriggers(List maliciousIocs, + List iocFindings, + IocScanContext iocScanContext, + List data, IocLookupDtos iocLookupDtos, + BiConsumer, Exception> triggerResultConsumer); + + abstract void matchAgainstThreatIntelAndReturnMaliciousIocs( + Map> iocsPerType, + Monitor monitor, + BiConsumer, Exception> callback, + Map> iocTypeToIndices); + + /** + * For each doc, we extract different maps for quick look up - + * 1. map of iocs as key to ioc type + * 2. ioc value to doc ids containing the ioc + * 4. doc id to iocs map (reverse mapping of 2) + */ + private IocLookupDtos extractIocsPerType + (List data, IocScanContext context) { + Map> iocsPerIocTypeMap = new HashMap<>(); + Map> iocValueToDocIdMap = new HashMap<>(); + Map> docIdToIocsMap = new HashMap<>(); + for (Data datum : data) { + for (PerIocTypeScanInput iocTypeToIndexFieldMapping : context.getThreatIntelInput().getPerIocTypeScanInputList()) { + String iocType = iocTypeToIndexFieldMapping.getIocType().toLowerCase(); + String concreteIndex = getIndexName(datum); + if (context.getConcreteIndexToMonitorInputIndicesMap().containsKey(concreteIndex)) { + // if concrete index resolves to multiple monitor input indices, it's undesirable. We just pick any one of the monitor input indices to get fields for each ioc. + String index = context.getConcreteIndexToMonitorInputIndicesMap().get(concreteIndex).get(0); + List fieldsConfiguredInMonitorForCurrentIndex = iocTypeToIndexFieldMapping.getIndexToFieldsMap().get(index); + if(fieldsConfiguredInMonitorForCurrentIndex != null && false == fieldsConfiguredInMonitorForCurrentIndex.isEmpty()) { + for (String field : fieldsConfiguredInMonitorForCurrentIndex) { + List vals = getValuesAsStringList(datum, field); + String id = getId(datum); + String docId = id + ":" + index; + Set iocs = docIdToIocsMap.getOrDefault(docId, new HashSet<>()); + iocs.addAll(vals); + docIdToIocsMap.put(docId, iocs); + for (String ioc : vals) { + Set docIds = iocValueToDocIdMap.getOrDefault(ioc, new HashSet<>()); + docIds.add(docId); + iocValueToDocIdMap.put(ioc, docIds); + } + if (false == vals.isEmpty()) { + iocs = iocsPerIocTypeMap.getOrDefault(iocType, new HashSet<>()); + iocs.addAll(vals); + iocsPerIocTypeMap.put(iocType, iocs); + } + } + } + } + } + } + return new IocLookupDtos(iocsPerIocTypeMap, iocValueToDocIdMap, docIdToIocsMap); + } + + abstract List getValuesAsStringList(Data datum, String field); + + abstract String getIndexName(Data datum); + + abstract String getId(Data datum); + + private void createIocFindings(List iocs, + Map> iocValueToDocIdMap, + IocScanContext iocScanContext, + BiConsumer, Exception> callback) { + try { + Instant timestamp = Instant.now(); + Monitor monitor = iocScanContext.getMonitor(); + // Map to collect unique IocValue with their respective FeedIds + Map> iocValueToFeedIds = new HashMap<>(); + Map iocValueToType = new HashMap<>(); + for (STIX2IOC ioc : iocs) { + String iocValue = ioc.getValue(); + if (false == iocValueToType.containsKey(iocValue)) + iocValueToType.put(iocValue, ioc.getType().toString()); + iocValueToFeedIds + .computeIfAbsent(iocValue, k -> new HashSet<>()) + .add(new IocWithFeeds(ioc.getId(), ioc.getFeedId(), ioc.getFeedName(), "")); //todo figure how to store index + } + + List iocFindings = new ArrayList<>(); + + for (Map.Entry> entry : iocValueToFeedIds.entrySet()) { + String iocValue = entry.getKey(); + Set iocWithFeeds = entry.getValue(); + + List relatedDocIds = new ArrayList<>(iocValueToDocIdMap.getOrDefault(iocValue, new HashSet<>())); + List feedIdsList = new ArrayList<>(iocWithFeeds); + try { + IocFinding iocFinding = new IocFinding( + UUID.randomUUID().toString(), // Generating a unique ID + relatedDocIds, + feedIdsList, // update to object + monitor.getId(), + monitor.getName(), + iocValue, + iocValueToType.get(iocValue), + timestamp, + UUID.randomUUID().toString() // TODO execution ID + ); + iocFindings.add(iocFinding); + } catch (Exception e) { + log.error(String.format("skipping creating ioc finding for %s due to unexpected failure.", entry.getKey()), e); + } + } + saveIocFindings(iocFindings, callback, monitor); + } catch (Exception e) { + log.error(() -> new ParameterizedMessage("Failed to create ioc findinges due to unexpected error {}", iocScanContext.getMonitor().getId()), e); + callback.accept(null, e); + } + } + + abstract void saveIocFindings + (List iocs, BiConsumer, Exception> callback, Monitor monitor); + + abstract void saveAlerts(List updatedAlerts, List newAlerts, Monitor monitor, BiConsumer, Exception> callback); + + protected static class IocLookupDtos { + private final Map> iocsPerIocTypeMap; + private final Map> iocValueToDocIdMap; + private final Map> docIdToIocsMap; + + public IocLookupDtos(Map> iocsPerIocTypeMap, Map> iocValueToDocIdMap, Map> docIdToIocsMap) { + this.iocsPerIocTypeMap = iocsPerIocTypeMap; + this.iocValueToDocIdMap = iocValueToDocIdMap; + this.docIdToIocsMap = docIdToIocsMap; + } + + public Map> getIocsPerIocTypeMap() { + return iocsPerIocTypeMap; + } + + public Map> getIocValueToDocIdMap() { + return iocValueToDocIdMap; + } + + public Map> getDocIdToIocsMap() { + return docIdToIocsMap; + } + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanServiceInterface.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanServiceInterface.java new file mode 100644 index 000000000..1826824d3 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanServiceInterface.java @@ -0,0 +1,13 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.service; + +import org.opensearch.securityanalytics.threatIntel.iocscan.dto.IocScanContext; + +import java.util.function.BiConsumer; + +public interface IoCScanServiceInterface { + + void scanIoCs( + IocScanContext iocScanContext, + BiConsumer scanCallback + ); +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java new file mode 100644 index 000000000..8a3c4a206 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java @@ -0,0 +1,515 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.service; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.ShardSearchFailure; +import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.document.DocumentField; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.commons.alerting.model.Monitor; +import org.opensearch.commons.alerting.model.Trigger; +import org.opensearch.commons.alerting.model.action.Action; +import org.opensearch.commons.alerting.model.remote.monitors.RemoteMonitorTrigger; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.TermsQueryBuilder; +import org.opensearch.search.SearchHit; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.commons.model.STIX2; +import org.opensearch.securityanalytics.correlation.alert.notifications.NotificationService; +import org.opensearch.securityanalytics.model.STIX2IOC; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; +import org.opensearch.securityanalytics.model.threatintel.ThreatIntelAlert; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.ThreatIntelAlertService; +import org.opensearch.securityanalytics.threatIntel.iocscan.dto.IocScanContext; +import org.opensearch.securityanalytics.threatIntel.model.monitor.ThreatIntelTrigger; +import org.opensearch.securityanalytics.threatIntel.model.monitor.TransportThreatIntelMonitorFanOutAction.SearchHitsOrException; +import org.opensearch.securityanalytics.threatIntel.util.ThreatIntelMonitorUtils; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; + +import static java.util.Collections.emptyList; +import static org.opensearch.securityanalytics.threatIntel.util.ThreatIntelMonitorUtils.getThreatIntelTriggerFromBytesReference; + +public class SaIoCScanService extends IoCScanService { + + private static final Logger log = LogManager.getLogger(SaIoCScanService.class); + private final Client client; + private final ClusterService clusterService; + private final NamedXContentRegistry xContentRegistry; + private final IocFindingService iocFindingService; + private final ThreatIntelAlertService threatIntelAlertService; + private final NotificationService notificationService; + + public SaIoCScanService(Client client, ClusterService clusterService, NamedXContentRegistry xContentRegistry, IocFindingService iocFindingService, + ThreatIntelAlertService threatIntelAlertService, NotificationService notificationService) { + this.client = client; + this.clusterService = clusterService; + this.xContentRegistry = xContentRegistry; + this.iocFindingService = iocFindingService; + this.threatIntelAlertService = threatIntelAlertService; + this.notificationService = notificationService; + } + + @Override + void executeTriggers(List maliciousIocs, List iocFindings, IocScanContext iocScanContext, List searchHits, IoCScanService.IocLookupDtos iocLookupDtos, BiConsumer, Exception> triggerResultConsumer) { + Monitor monitor = iocScanContext.getMonitor(); + if (maliciousIocs.isEmpty() || monitor.getTriggers().isEmpty()) { + triggerResultConsumer.accept(Collections.emptyList(), null); + return; + } + initAlertsIndex( + ActionListener.wrap( + r -> { + GroupedActionListener> allTriggerResultListener = getGroupedListenerForAllTriggersResponse(iocScanContext.getMonitor(), + triggerResultConsumer); + for (Trigger trigger : monitor.getTriggers()) { + executeTrigger(iocFindings, trigger, monitor, allTriggerResultListener); + } + }, + e -> { + log.error(() -> new ParameterizedMessage( + "Threat intel monitor {} Failed to execute triggers . Failed to initialize threat intel alerts index", + monitor.getId()), e); + triggerResultConsumer.accept(Collections.emptyList(), null); + } + ) + ); + } + + private void executeTrigger(List iocFindings, + Trigger trigger, + Monitor monitor, + ActionListener> listener) { + try { + RemoteMonitorTrigger remoteMonitorTrigger = (RemoteMonitorTrigger) trigger; + ThreatIntelTrigger threatIntelTrigger = getThreatIntelTriggerFromBytesReference(remoteMonitorTrigger, xContentRegistry); + ArrayList triggerMatchedFindings = ThreatIntelMonitorUtils.getTriggerMatchedFindings(iocFindings, threatIntelTrigger); + if (triggerMatchedFindings.isEmpty()) { + log.debug("Threat intel monitor {} no matches for trigger {}", monitor.getId(), trigger.getName()); + listener.onResponse(emptyList()); + } else { + fetchExistingAlertsForTrigger(monitor, triggerMatchedFindings, trigger, ActionListener.wrap( + existingAlerts -> { + saveAlertsAndExecuteActions(iocFindings, trigger, monitor, existingAlerts, triggerMatchedFindings, threatIntelTrigger, listener); + }, + e -> { + log.error(() -> new ParameterizedMessage( + "Threat intel monitor {} Failed to execute trigger {}. Failure while fetching existing alerts", + monitor.getId(), trigger.getName()), e); + listener.onFailure(e); + } + )); + } + } catch (Exception e) { + log.error(() -> new ParameterizedMessage( + "Threat intel monitor {} Failed to execute trigger {}", monitor.getId(), trigger.getName()), + e + ); + listener.onFailure(e); + } + } + + private void saveAlertsAndExecuteActions(List iocFindings, + Trigger trigger, + Monitor monitor, + List existingAlerts, + ArrayList triggerMatchedFindings, + ThreatIntelTrigger threatIntelTrigger, ActionListener> listener) { + Map iocToUpdatedAlertsMap = ThreatIntelMonitorUtils.prepareAlertsToUpdate(triggerMatchedFindings, existingAlerts); + List newAlerts = ThreatIntelMonitorUtils.prepareNewAlerts(monitor, trigger, triggerMatchedFindings, iocToUpdatedAlertsMap); + ThreatIntelAlertContext ctx = new ThreatIntelAlertContext(threatIntelTrigger, + trigger, + iocFindings, + monitor, + newAlerts, + existingAlerts); + if (false == trigger.getActions().isEmpty()) { + saveAlerts(new ArrayList<>(iocToUpdatedAlertsMap.values()), + newAlerts, + monitor, + (threatIntelAlerts, e) -> { + if (e != null) { + log.error(String.format("Threat intel monitor %s: Failed to save alerts for trigger %s", monitor.getId(), trigger.getId()), e); + listener.onFailure(e); + } else { + GroupedActionListener notifsListener = new GroupedActionListener<>(ActionListener.wrap( + r -> { + listener.onResponse(threatIntelAlerts); + }, ex -> { + log.error(String.format("Threat intel monitor {}: Failed to send notification for trigger {}", monitor.getId(), trigger.getId()), ex); + listener.onFailure(new SecurityAnalyticsException("Failed to send notification", RestStatus.INTERNAL_SERVER_ERROR, ex)); + } + ), trigger.getActions().size()); + + for (Action action : trigger.getActions()) { + try { + String transformedSubject = NotificationService.compileTemplate(ctx, action.getSubjectTemplate()); + String transformedMessage = NotificationService.compileTemplate(ctx, action.getMessageTemplate()); + String configId = action.getDestinationId(); + notificationService.sendNotification(configId, trigger.getSeverity(), transformedSubject, transformedMessage, notifsListener); + } catch (Exception ex) { + log.error(String.format("Threat intel monitor %s: Failed to send notification to %s for trigger %s", monitor.getId(), action.getDestinationId(), trigger.getId()), ex); + notifsListener.onFailure(new SecurityAnalyticsException("Failed to send notification", RestStatus.INTERNAL_SERVER_ERROR, ex)); + } + + } + } + }); + + } else { + saveAlerts(new ArrayList<>(iocToUpdatedAlertsMap.values()), + newAlerts, + monitor, + (threatIntelAlerts, e) -> { + if (e != null) { + log.error(String.format("Threat intel monitor %s: Failed to save alerts for trigger %s", monitor.getId(), trigger.getId()), e); + listener.onFailure(e); + } else { + listener.onResponse(threatIntelAlerts); + } + }); + } + } + + private void fetchExistingAlertsForTrigger(Monitor monitor, + ArrayList findings, + Trigger trigger, + ActionListener> listener) { + if (findings.isEmpty()) { + listener.onResponse(emptyList()); + return; + } + SearchSourceBuilder ssb = ThreatIntelMonitorUtils.getSearchSourceBuilderForExistingAlertsQuery(findings, trigger); + threatIntelAlertService.search(ssb, ActionListener.wrap( + searchResponse -> { + List alerts = new ArrayList<>(); + if (searchResponse.getHits() == null || searchResponse.getHits().getHits() == null) { + listener.onResponse(alerts); + return; + } + for (SearchHit hit : searchResponse.getHits().getHits()) { + XContentParser xcp = XContentType.JSON.xContent().createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString() + ); + if(xcp.currentToken() == null) + xcp.nextToken(); + ThreatIntelAlert alert = ThreatIntelAlert.parse(xcp, hit.getVersion()); + alerts.add(alert); + } + listener.onResponse(alerts); + }, + e -> { + log.error(() -> new ParameterizedMessage( + "Threat intel monitor {} Failed to execute trigger {}. Unexpected error in fetching existing alerts for dedupe", monitor.getId(), trigger.getName()), + e + ); + listener.onFailure(e); + } + )); + } + + private GroupedActionListener> getGroupedListenerForAllTriggersResponse(Monitor monitor, BiConsumer, Exception> triggerResultConsumer) { + return new GroupedActionListener<>(ActionListener.wrap( + r -> { + List list = new ArrayList<>(); + r.forEach(list::addAll); + triggerResultConsumer.accept(list, null); + }, e -> { + log.error(() -> new ParameterizedMessage( + "Threat intel monitor {} Failed to execute triggers {}", monitor.getId()), + e + ); + triggerResultConsumer.accept(emptyList(), e); + } + ), monitor.getTriggers().size()); + } + + @Override + void matchAgainstThreatIntelAndReturnMaliciousIocs( + Map> iocsPerType, + Monitor monitor, + BiConsumer, Exception> callback, + Map> iocTypeToIndices) { + long startTime = System.currentTimeMillis(); + int numIocs = iocsPerType.values().stream().mapToInt(Set::size).sum(); + GroupedActionListener groupedListenerForAllIocTypes = getGroupedListenerForIocScanFromAllIocTypes(iocsPerType, monitor, callback, startTime, numIocs); + for (String iocType : iocsPerType.keySet()) { + List indices = iocTypeToIndices.get(iocType); + Set iocs = iocsPerType.get(iocType); + if (iocTypeToIndices.containsKey(iocType.toLowerCase())) { + if (indices.isEmpty()) { + log.debug( + "Threat intel monitor {} : No ioc indices of type {} found so no scan performed.", + monitor.getId(), + iocType + ); + groupedListenerForAllIocTypes.onResponse(new SearchHitsOrException(emptyList(), null)); + } else if (iocs.isEmpty()) { + log.debug( + "Threat intel monitor {} : No iocs of type {} found in user data so no scan performed.", + monitor.getId(), + iocType + ); + groupedListenerForAllIocTypes.onResponse(new SearchHitsOrException(emptyList(), null)); + } else { + performScanForMaliciousIocsPerIocType(indices, iocs, monitor, iocType, groupedListenerForAllIocTypes); + } + } else { + groupedListenerForAllIocTypes.onResponse(new SearchHitsOrException(emptyList(), null)); + } + } + } + + private GroupedActionListener getGroupedListenerForIocScanFromAllIocTypes(Map> iocsPerType, Monitor monitor, BiConsumer, Exception> callback, long startTime, int numIocs) { + return new GroupedActionListener<>( + ActionListener.wrap( + lists -> { + long endTime = System.currentTimeMillis(); + long timetaken = endTime - startTime; + log.debug("IOC_SCAN: Threat intel monitor {} completed Ioc match phase in {} millis for {} iocs", + monitor.getId(), timetaken, numIocs); + List hits = new ArrayList<>(); + lists.forEach(hitsOrException -> + hits.addAll(hitsOrException.getHits() == null ? + emptyList() : + hitsOrException.getHits())); + List iocs = new ArrayList<>(); + hits.forEach(hit -> { + try { + XContentParser xcp = XContentType.JSON.xContent().createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, + hit.getSourceAsString()); + xcp.nextToken(); + + STIX2IOC ioc = STIX2IOC.parse(xcp, hit.getId(), hit.getVersion()); + iocs.add(ioc); + } catch (Exception e) { + log.error(() -> new ParameterizedMessage( + "Failed to parse IOC doc from hit {} index {}", hit.getId(), hit.getIndex()), + e + ); + } + }); + callback.accept(iocs, null); + }, + e -> { + log.error("Threat intel monitor {} :Unexpected error while scanning data for malicious Iocs", e); + callback.accept(emptyList(), e); + } + ), + iocsPerType.size() + ); + } + + private void performScanForMaliciousIocsPerIocType( + List indices, + Set iocs, + Monitor monitor, + String iocType, + GroupedActionListener listener) { + // TODO change ioc indices max terms count to 100k and experiment + // TODO add fuzzy postings on ioc value field to enable bloomfilter on iocs as an index data structure and benchmark performance + int maxTerms = clusterService.getClusterSettings().get(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT); + GroupedActionListener perIocTypeListener = getGroupedListenerForIocScanPerIocType(iocs, monitor, iocType, listener, maxTerms); + List iocList = new ArrayList<>(iocs); + int totalIocs = iocList.size(); + + for (int start = 0; start < totalIocs; start += maxTerms) { + int end = Math.min(start + maxTerms, totalIocs); + List iocsSublist = iocList.subList(start, end); + SearchRequest searchRequest = getSearchRequestForIocType(indices, iocType, iocsSublist); + client.search(searchRequest, ActionListener.wrap( + searchResponse -> { + if (searchResponse.isTimedOut()) { + log.error("Threat intel monitor {} scan with {} user data indicators TIMED OUT for ioc Type {}", + monitor.getId(), + iocsSublist.size(), + iocType + ); + } + if (searchResponse.getFailedShards() > 0) { + for (ShardSearchFailure shardFailure : searchResponse.getShardFailures()) { + log.error("Threat intel monitor {} scan with {} user data indicators for ioc Type {} has Shard failures {}", + monitor.getId(), + iocsSublist.size(), + iocType, + shardFailure.toString() + ); + } + } + perIocTypeListener.onResponse(new SearchHitsOrException( + searchResponse.getHits() == null || searchResponse.getHits().getHits() == null ? + emptyList() : Arrays.asList(searchResponse.getHits().getHits()), null)); + }, + e -> { + log.error(() -> new ParameterizedMessage("Threat intel monitor {} scan with {} user data indicators failed for ioc Type {}", + monitor.getId(), + iocsSublist.size(), + iocType), e + ); + perIocTypeListener.onResponse(new SearchHitsOrException(emptyList(), e)); + } + )); + } + } + + private static SearchRequest getSearchRequestForIocType(List indices, String iocType, List iocsSublist) { + SearchRequest searchRequest = new SearchRequest(indices.toArray(new String[0])); + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + // add the iocs sublist + boolQueryBuilder.must(new TermsQueryBuilder(STIX2.VALUE_FIELD, iocsSublist)); + // add ioc type filter + boolQueryBuilder.must(new TermsQueryBuilder(STIX2.TYPE_FIELD, iocType.toLowerCase(Locale.ROOT))); + searchRequest.source().query(boolQueryBuilder); + return searchRequest; + } + + /** + * grouped listener for a given ioc type to listen and collate malicious iocs in search hits from batched search calls. + * batching done for every 65536 or MAX_TERMS setting number of iocs in a list. + */ + private GroupedActionListener getGroupedListenerForIocScanPerIocType(Set iocs, Monitor monitor, String iocType, GroupedActionListener groupedListenerForAllIocTypes, int maxTerms) { + return new GroupedActionListener<>( + ActionListener.wrap( + (Collection searchHitsOrExceptions) -> { + if (false == searchHitsOrExceptions.stream().allMatch(shoe -> shoe.getException() != null)) { + List searchHits = new ArrayList<>(); + searchHitsOrExceptions.forEach(searchHitsOrException -> { + if (searchHitsOrException.getException() != null) { + log.error( + () -> new ParameterizedMessage( + "Threat intel monitor {}: Failed to perform ioc scan on one batch for ioc type : ", + monitor.getId(), iocType), searchHitsOrException.getException()); + } else { + searchHits.addAll(searchHitsOrException.getHits() != null ? + searchHitsOrException.getHits() : emptyList()); + } + }); + // we collect all hits we can and log all exceptions and submit to outer listener + groupedListenerForAllIocTypes.onResponse(new SearchHitsOrException(searchHits, null)); + } else { + // we collect all exceptions under one exception and respond to outer listener + groupedListenerForAllIocTypes.onResponse(new SearchHitsOrException(emptyList(), buildException(searchHitsOrExceptions)) + ); + } + }, e -> { + log.error( + () -> new ParameterizedMessage( + "Threat intel monitor {}: Failed to perform ioc scan for ioc type : ", + monitor.getId(), iocType), e); + groupedListenerForAllIocTypes.onResponse(new SearchHitsOrException(emptyList(), e)); + } + ), + getGroupSizeForIocs(iocs, maxTerms) + ); + } + + private Exception buildException(Collection searchHitsOrExceptions) { + Exception e = null; + for (SearchHitsOrException searchHitsOrException : searchHitsOrExceptions) { + if (e == null) + e = searchHitsOrException.getException(); + else { + e.addSuppressed(searchHitsOrException.getException()); + } + } + return e; + } + + private static int getGroupSizeForIocs(Set iocs, int maxTerms) { + return iocs.size() / maxTerms + (iocs.size() % maxTerms == 0 ? 0 : 1); + } + + @Override + public List getValuesAsStringList(SearchHit hit, String field) { + if (hit.getFields().containsKey(field)) { + DocumentField documentField = hit.getFields().get(field); + return documentField.getValues().stream().filter(Objects::nonNull).map(Object::toString).collect(Collectors.toList()); + } else return emptyList(); + } + + @Override + public String getIndexName(SearchHit hit) { + return hit.getIndex(); + } + + @Override + public String getId(SearchHit hit) { + return hit.getId(); + } + + @Override + void saveIocFindings(List iocFindings, BiConsumer, Exception> callback, Monitor monitor) { + if (iocFindings == null || iocFindings.isEmpty()) { + callback.accept(emptyList(), null); + return; + } + log.debug("Threat intel monitor {}: Indexing {} ioc findings", monitor.getId(), iocFindings.size()); + iocFindingService.bulkIndexEntities(iocFindings, ActionListener.wrap( + v -> { + callback.accept(iocFindings, null); + }, + e -> { + log.error( + () -> new ParameterizedMessage( + "Threat intel monitor {}: Failed to index ioc findings ", + monitor.getId()), e + ); + callback.accept(emptyList(), e); + } + )); + } + + @Override + void saveAlerts(List updatedAlerts, List newAlerts, Monitor monitor, BiConsumer, Exception> callback) { + if ((newAlerts == null || newAlerts.isEmpty()) && (updatedAlerts == null || updatedAlerts.isEmpty())) { + callback.accept(emptyList(), null); + return; + } + log.debug("Threat intel monitor {}: Indexing {} new threat intel alerts and updating {} existing alerts", monitor.getId(), newAlerts.size(), updatedAlerts.size()); + threatIntelAlertService.bulkIndexEntities(newAlerts, updatedAlerts, ActionListener.wrap( + v -> { + ArrayList threatIntelAlerts = new ArrayList<>(newAlerts); + threatIntelAlerts.addAll(updatedAlerts); + callback.accept(threatIntelAlerts, null); + }, + e -> { + log.error( + () -> new ParameterizedMessage( + "Threat intel monitor {}: Failed to index alerts ", + monitor.getId()), e + ); + callback.accept(emptyList(), e); + } + )); + } + + private void initAlertsIndex(ActionListener listener) { + threatIntelAlertService.createIndexIfNotExists(listener); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/ThreatIntelAlertContext.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/ThreatIntelAlertContext.java new file mode 100644 index 000000000..c2cdbaf65 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/ThreatIntelAlertContext.java @@ -0,0 +1,60 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.service; + +import org.opensearch.commons.alerting.model.Alert; +import org.opensearch.commons.alerting.model.Monitor; +import org.opensearch.commons.alerting.model.Trigger; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; +import org.opensearch.securityanalytics.model.threatintel.ThreatIntelAlert; +import org.opensearch.securityanalytics.threatIntel.model.monitor.ThreatIntelTrigger; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * context that stores information for sending threat intel monitor notification. + * It is available to use in Threat intel monitor runner in mustache template. + */ + +public class ThreatIntelAlertContext { + public static final String MONITOR_FIELD = "monitor"; + public static final String NEW_ALERTS_FIELD = "new_alerts"; + public static final String EXISTING_ALERTS_FIELD = "existing_alerts"; + + private final List dataSources; + private final List iocTypes; + private final String triggerName; + private final String triggerId; + private final List newAlerts; + private final List existingAlerts; + private final String severity; + private final List findingIds; + private final Monitor monitor; + + public ThreatIntelAlertContext(ThreatIntelTrigger threatIntelTrigger, Trigger trigger, List findingIds, Monitor monitor, List newAlerts, List existingAlerts) { + this.dataSources = threatIntelTrigger.getDataSources(); + this.iocTypes = threatIntelTrigger.getIocTypes(); + this.triggerName = trigger.getName(); + this.triggerId = trigger.getId(); + this.newAlerts = newAlerts; + this.existingAlerts = existingAlerts; + this.severity = triggerId; + this.findingIds = findingIds; + this.monitor = monitor; + } + + //cannot add trigger as Remote Trigger holds bytereference of object and not object itself + public Map asTemplateArg() { + return Map.of( + ThreatIntelTrigger.DATA_SOURCES, dataSources, + ThreatIntelTrigger.IOC_TYPES, iocTypes, + Trigger.NAME_FIELD, triggerName, + Trigger.ID_FIELD, triggerId, + Trigger.SEVERITY_FIELD, severity, + Alert.FINDING_IDS, findingIds.stream().map(IocFinding::asTemplateArg).collect(Collectors.toList()), + MONITOR_FIELD, monitor.asTemplateArg(), + NEW_ALERTS_FIELD, newAlerts.stream().map(ThreatIntelAlert::asTemplateArg).collect(Collectors.toList()), + EXISTING_ALERTS_FIELD, existingAlerts.stream().map(ThreatIntelAlert::asTemplateArg).collect(Collectors.toList()) + ); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/ThreatIntelMonitorRunner.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/ThreatIntelMonitorRunner.java new file mode 100644 index 000000000..f683a5ed9 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/ThreatIntelMonitorRunner.java @@ -0,0 +1,37 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.service; + +import org.opensearch.action.ActionType; + +import org.opensearch.alerting.spi.RemoteMonitorRunner; +import org.opensearch.commons.alerting.action.DocLevelMonitorFanOutResponse; + +public class ThreatIntelMonitorRunner extends RemoteMonitorRunner { + + public static final String FAN_OUT_ACTION_NAME = "cluster:admin/opensearch/securityanalytics/threatintel/monitors/fanout"; + public static final String THREAT_INTEL_MONITOR_TYPE = "ti_doc_level_monitor"; + + public static final String SAMPLE_REMOTE_DOC_LEVEL_MONITOR_RUNNER_INDEX = ".opensearch-alerting-sample-remote-doc-level-monitor"; + + public static final ActionType REMOTE_DOC_LEVEL_MONITOR_ACTION_INSTANCE = new ActionType<>(FAN_OUT_ACTION_NAME, + DocLevelMonitorFanOutResponse::new); + + private static ThreatIntelMonitorRunner INSTANCE; + + public static ThreatIntelMonitorRunner getMonitorRunner() { + if (INSTANCE != null) { + return INSTANCE; + } + synchronized (ThreatIntelMonitorRunner.class) { + if (INSTANCE != null) { + return INSTANCE; + } + INSTANCE = new ThreatIntelMonitorRunner(); + return INSTANCE; + } + } + + @Override + public String getFanOutAction() { + return FAN_OUT_ACTION_NAME; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java index 1d8d8643f..89dc729b4 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java @@ -11,21 +11,21 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.core.action.ActionListener; import org.opensearch.jobscheduler.spi.JobExecutionContext; -import org.opensearch.jobscheduler.spi.LockModel; import org.opensearch.jobscheduler.spi.ScheduledJobParameter; import org.opensearch.jobscheduler.spi.ScheduledJobRunner; -import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; import java.time.Instant; -import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; import org.opensearch.securityanalytics.threatIntel.action.ThreatIntelIndicesResponse; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService; import org.opensearch.threadpool.ThreadPool; /** @@ -115,10 +115,24 @@ protected Runnable updateJobRunner(final ScheduledJobParameter jobParameter) { ActionListener.wrap(lock -> { updateJobParameter(jobParameter, lockService.getRenewLockRunnable(new AtomicReference<>(lock)), ActionListener.wrap( - r -> lockService.releaseLock(lock), + r -> lockService.releaseLock(lock, ActionListener.wrap( + response -> { + log.debug("Released tif job parameter lock with id [{}]", lock.getLockId()); + }, + ex -> { + log.error(String.format("Unexpected failure while trying to release lock [%s] for tif job parameter [%s].", lock.getLockId(), jobParameter.getName()), ex); + } + )), e -> { log.error("Failed to update job parameter " + jobParameter.getName(), e); - lockService.releaseLock(lock); + lockService.releaseLock(lock, ActionListener.wrap( + response -> { + log.debug("Released tif job parameter lock with id [{}]", lock.getLockId()); + }, + ex -> { + log.error(String.format("Unexpected failure while trying to release lock [%s] for tif job parameter [%s].", lock.getLockId(), jobParameter.getName()), ex); + } + )); } )); }, e -> { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFSourceConfigRunner.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFSourceConfigRunner.java new file mode 100644 index 000000000..1c098cfc9 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFSourceConfigRunner.java @@ -0,0 +1,149 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.jobscheduler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.core.action.ActionListener; +import org.opensearch.extensions.AcknowledgedResponse; +import org.opensearch.jobscheduler.spi.JobExecutionContext; +import org.opensearch.jobscheduler.spi.ScheduledJobParameter; +import org.opensearch.jobscheduler.spi.ScheduledJobRunner; +import org.opensearch.jobscheduler.spi.utils.LockService; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService; +import org.opensearch.threadpool.ThreadPool; + +import java.util.concurrent.atomic.AtomicReference; + +/** + * This is a background task which is responsible for updating threat intel feed iocs and the source config + */ +public class TIFSourceConfigRunner implements ScheduledJobRunner { + private static final Logger log = LogManager.getLogger(TIFSourceConfigRunner.class); + private static TIFSourceConfigRunner INSTANCE; + public static TIFSourceConfigRunner getJobRunnerInstance() { + if (INSTANCE != null) { + return INSTANCE; + } + synchronized (TIFSourceConfigRunner.class) { + if (INSTANCE != null) { + return INSTANCE; + } + INSTANCE = new TIFSourceConfigRunner(); + return INSTANCE; + } + } + + private ClusterService clusterService; + private TIFLockService lockService; + private boolean initialized; + private ThreadPool threadPool; + private SATIFSourceConfigManagementService saTifSourceConfigManagementService; + private SATIFSourceConfigService saTifSourceConfigService; + + private TIFSourceConfigRunner() { + // Singleton class, use getJobRunner method instead of constructor + } + + public void initialize( + final ClusterService clusterService, + final TIFLockService threatIntelLockService, + final ThreadPool threadPool, + final SATIFSourceConfigManagementService saTifSourceConfigManagementService, + final SATIFSourceConfigService saTifSourceConfigService + ) { + this.clusterService = clusterService; + this.lockService = threatIntelLockService; + this.threadPool = threadPool; + this.initialized = true; + this.saTifSourceConfigManagementService = saTifSourceConfigManagementService; + this.saTifSourceConfigService = saTifSourceConfigService; + } + + @Override + public void runJob(final ScheduledJobParameter jobParameter, final JobExecutionContext context) { + if (initialized == false) { + throw new AssertionError("This instance is not initialized"); + } + + if (jobParameter instanceof SATIFSourceConfig == false) { + log.error("Illegal state exception: job parameter is not instance of TIF Source Config"); + throw new IllegalStateException( + "job parameter is not instance of TIF Source Config, type: " + jobParameter.getClass().getCanonicalName() + ); + } + + if (this.clusterService == null) { + throw new IllegalStateException("ClusterService is not initialized."); + } + + if (this.threadPool == null) { + throw new IllegalStateException("ThreadPool is not initialized."); + } + final LockService lockService = context.getLockService(); // todo + threadPool.generic().submit(retrieveLockAndUpdateConfig((SATIFSourceConfig)jobParameter)); + } + + /** + * Update threat intel feed config and data + * + * Lock is used so that only one of nodes run this task. + * + * @param saTifSourceConfig the TIF source config that is scheduled onto the job scheduler + */ + protected Runnable retrieveLockAndUpdateConfig(final SATIFSourceConfig saTifSourceConfig) { + log.info("Update job started for a TIF Source Config [{}]", saTifSourceConfig.getId()); + + return () -> lockService.acquireLock( + saTifSourceConfig.getId(), + TIFLockService.LOCK_DURATION_IN_SECONDS, + ActionListener.wrap(lock -> { + updateSourceConfigAndIOCs(saTifSourceConfig, lockService.getRenewLockRunnable(new AtomicReference<>(lock)), + ActionListener.wrap( + r -> { + lockService.releaseLock(lock, ActionListener.wrap( + response -> { + log.debug("Released threat intel source config lock with id [{}]", lock.getLockId()); + }, + ex -> { + log.error(String.format("Unexpected failure while trying to release lock [%s] for tif source config [%s].", lock.getLockId(), saTifSourceConfig.getId()), ex); + } + )); + }, + e -> { + log.error("Failed to update threat intel source config " + saTifSourceConfig.getName(), e); + lockService.releaseLock(lock, ActionListener.wrap( + response -> { + log.debug("Released threat intel source config lock with id [{}]", lock.getLockId()); + }, + ex -> { + log.error(String.format("Unexpected failure while trying to release lock [%s] for tif source config [%s].", lock.getLockId(), saTifSourceConfig.getId()), ex); + } + )); + } + )); + }, e -> { + log.error("Failed to update. Another processor is holding a lock for job parameter[{}]", saTifSourceConfig.getName()); + }) + ); + } + + protected void updateSourceConfigAndIOCs(final SATIFSourceConfig SaTifSourceConfig, final Runnable renewLock, ActionListener listener) { + saTifSourceConfigManagementService.refreshTIFSourceConfig(SaTifSourceConfig.getId(), null, ActionListener.wrap( + r -> { + log.info("Successfully updated source config and IOCs for threat intel source config [{}]", SaTifSourceConfig.getId()); + listener.onResponse(new AcknowledgedResponse(true)); + }, e -> { + log.error("Failed to update source config and IOCs for threat intel source config [{}]", SaTifSourceConfig.getId()); + listener.onFailure(e); + } + )); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/DefaultIocStoreConfig.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/DefaultIocStoreConfig.java new file mode 100644 index 000000000..a63bc99d3 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/DefaultIocStoreConfig.java @@ -0,0 +1,175 @@ +package org.opensearch.securityanalytics.threatIntel.model; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.securityanalytics.commons.model.IOCType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Model used for the default IOC store configuration + * Stores the IOC mapping in a list of IocToIndexDetails which contains the ioc type, index pattern, and the active index + */ +public class DefaultIocStoreConfig extends IocStoreConfig implements Writeable, ToXContent { + private static final Logger log = LogManager.getLogger(DefaultIocStoreConfig.class); + public static final String DEFAULT_FIELD = "default"; + public static final String IOC_TO_INDEX_DETAILS_FIELD = "ioc_to_index_details"; + private final List iocToIndexDetails; + + public DefaultIocStoreConfig(List iocToIndexDetails) { + this.iocToIndexDetails = iocToIndexDetails; + } + + public DefaultIocStoreConfig(StreamInput sin) throws IOException { + this.iocToIndexDetails = Collections.unmodifiableList(sin.readList(IocToIndexDetails::new)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeCollection(iocToIndexDetails); + } + + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject() + .field(DEFAULT_FIELD); + builder.startObject() + .field(IOC_TO_INDEX_DETAILS_FIELD, iocToIndexDetails); + builder.endObject(); + builder.endObject(); + return builder; + } + + public static DefaultIocStoreConfig parse(XContentParser xcp) throws IOException { + List iocToIndexDetails = null; + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + switch (fieldName) { + case DEFAULT_FIELD: + break; + case IOC_TO_INDEX_DETAILS_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + iocToIndexDetails = null; + } else { + iocToIndexDetails = new ArrayList<>(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + iocToIndexDetails.add(IocToIndexDetails.parse(xcp)); + } + } + break; + default: + xcp.skipChildren(); + } + } + return new DefaultIocStoreConfig(iocToIndexDetails); + } + + @Override + public String name() { + return DEFAULT_FIELD; + } + + public List getIocToIndexDetails() { + return iocToIndexDetails; + } + + public static class IocToIndexDetails implements Writeable, ToXContent { + public static final String IOC_TYPE_FIELD = "ioc_type"; + public static final String INDEX_PATTERN_FIELD = "index_pattern"; + public static final String ACTIVE_INDEX_FIELD = "active_index"; + private final IOCType iocType; + private final String indexPattern; + private final String activeIndex; + + public IocToIndexDetails(IOCType iocType, String indexPattern, String activeIndex) { + this.iocType = iocType; + this.indexPattern = indexPattern; + this.activeIndex = activeIndex; + } + + public IocToIndexDetails(StreamInput sin) throws IOException { + this( + new IOCType(sin.readString()), + sin.readString(), + sin.readString() + ); + } + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(iocType.toString()); + out.writeString(indexPattern); + out.writeString(activeIndex); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field(IOC_TYPE_FIELD, iocType.toString()) + .field(INDEX_PATTERN_FIELD, indexPattern) + .field(ACTIVE_INDEX_FIELD, activeIndex) + .endObject(); + } + + public static IocToIndexDetails parse(XContentParser xcp) throws IOException { + IOCType iocType = null; + String indexPattern = null; + String activeIndex = null; + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case IOC_TYPE_FIELD: + iocType = toIocType(xcp.text()); + break; + case INDEX_PATTERN_FIELD: + indexPattern = xcp.text(); + break; + case ACTIVE_INDEX_FIELD: + activeIndex = xcp.text(); + break; + default: + xcp.skipChildren(); + } + } + return new IocToIndexDetails(iocType, indexPattern, activeIndex); + } + + public static IOCType toIocType(String name) { + try { + return new IOCType(name); + } catch (IllegalArgumentException e) { + log.error("Invalid Ioc type, cannot be parsed.", e); + return null; + } + } + + public IOCType getIocType() { + return iocType; + } + + public String getIndexPattern() { + return indexPattern; + } + + public String getActiveIndex() { + return activeIndex; + } + + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/IocStoreConfig.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/IocStoreConfig.java new file mode 100644 index 000000000..58675cfea --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/IocStoreConfig.java @@ -0,0 +1,58 @@ +package org.opensearch.securityanalytics.threatIntel.model; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Locale; + +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; + +/** + * Base class for the IOC store config that other implementations will extend from + */ +public abstract class IocStoreConfig { + private static final Logger log = LogManager.getLogger(IocStoreConfig.class); + abstract String name(); + static IocStoreConfig readFrom(StreamInput sin) throws IOException { + Type type = sin.readEnum(Type.class); + switch(type) { + case DEFAULT: + return new DefaultIocStoreConfig(sin); + default: + throw new IllegalStateException("Unexpected input [" + type + "] when reading ioc store config"); + } + } + + static IocStoreConfig parse(XContentParser xcp) throws IOException { + IocStoreConfig iocStoreConfig = null; + + ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + switch (fieldName) { + case "default": + iocStoreConfig = DefaultIocStoreConfig.parse(xcp); + break; + } + } + + return iocStoreConfig; + } + + public void writeTo(StreamOutput out) throws IOException {} + + + enum Type { + DEFAULT(); + @Override + public String toString() { + return this.name().toLowerCase(Locale.ROOT); + } + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/IocUploadSource.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/IocUploadSource.java new file mode 100644 index 000000000..865120fac --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/IocUploadSource.java @@ -0,0 +1,99 @@ +package org.opensearch.securityanalytics.threatIntel.model; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.securityanalytics.model.STIX2IOCDto; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class IocUploadSource extends Source implements Writeable, ToXContent { + public static final String IOCS_FIELD = "iocs"; + public static final String FILE_NAME_FIELD = "file_name"; + private String fileName; + private List iocs; + + public IocUploadSource(String fileName, List iocs) { + this.fileName = fileName; + this.iocs = iocs; + } + + public IocUploadSource(StreamInput sin) throws IOException { + this ( + sin.readOptionalString(), // file name + Collections.unmodifiableList(sin.readList(STIX2IOCDto::new)) // iocs + ); + } + + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalString(fileName); + out.writeCollection(iocs); + } + + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.startObject(IOC_UPLOAD_FIELD); + if (fileName == null) { + builder.nullField(FILE_NAME_FIELD); + } else { + builder.field(FILE_NAME_FIELD, fileName); + } + builder.field(IOCS_FIELD, iocs); + builder.endObject(); + builder.endObject(); + return builder; + } + + @Override + String name() { + return IOC_UPLOAD_FIELD; + } + + public static IocUploadSource parse(XContentParser xcp) throws IOException { + String fileName = null; + List iocs = null; + + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + switch (fieldName) { + case FILE_NAME_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + fileName = null; + } else { + fileName = xcp.text(); + } + break; + case IOCS_FIELD: + iocs = new ArrayList<>(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + iocs.add(STIX2IOCDto.parse(xcp, null, null)); + } + break; + default: + break; + } + } + return new IocUploadSource(fileName, iocs); + } + + public List getIocs() { + return iocs; + } + + public void setIocs(List iocs) { + this.iocs = iocs; + } + + public String getFileName() { + return fileName; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/S3Source.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/S3Source.java new file mode 100644 index 000000000..abe23500b --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/S3Source.java @@ -0,0 +1,130 @@ +package org.opensearch.securityanalytics.threatIntel.model; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; + +import java.io.IOException; + +public class S3Source extends Source implements Writeable, ToXContent { + + public static final String BUCKET_NAME_FIELD = "bucket_name"; + public static final String OBJECT_KEY_FIELD = "object_key"; + public static final String REGION_FIELD = "region"; + public static final String ROLE_ARN_FIELD = "role_arn"; + private String bucketName; + private String objectKey; + private String region; + private String roleArn; + + public S3Source(String bucketName, String objectKey, String region, String roleArn) { + this.bucketName = bucketName; + this.objectKey = objectKey; + this.region = region; + this.roleArn = roleArn; + } + + public S3Source(StreamInput sin) throws IOException { + this ( + sin.readString(), // bucket name + sin.readString(), // object key + sin.readString(), // region + sin.readString() // role arn + ); + } + + public void writeTo(StreamOutput out) throws IOException { + out.writeString(bucketName); + out.writeString(objectKey); + out.writeString(region); + out.writeString(roleArn); + } + + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject() + .field(S3_FIELD); + builder.startObject() + .field(BUCKET_NAME_FIELD, bucketName) + .field(OBJECT_KEY_FIELD, objectKey) + .field(REGION_FIELD, region) + .field(ROLE_ARN_FIELD, roleArn); + builder.endObject(); + builder.endObject(); + return builder; + } + + @Override + String name() { + return S3_FIELD; + } + + public static S3Source parse(XContentParser xcp) throws IOException { + String bucketName = null; + String objectKey = null; + String region = null; + String roleArn = null; + + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + switch (fieldName) { + case BUCKET_NAME_FIELD: + bucketName = xcp.text(); + break; + case OBJECT_KEY_FIELD: + objectKey = xcp.text(); + break; + case REGION_FIELD: + region = xcp.text(); + break; + case ROLE_ARN_FIELD: + roleArn = xcp.text(); + break; + default: + break; + } + } + return new S3Source( + bucketName, + objectKey, + region, + roleArn + ); + } + + public String getBucketName() { + return bucketName; + } + + public void setBucketName(String bucketName) { + this.bucketName = bucketName; + } + + public String getObjectKey() { + return objectKey; + } + + public void setObjectKey(String objectKey) { + this.objectKey = objectKey; + } + + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + public String getRoleArn() { + return roleArn; + } + + public void setRoleArn(String roleArn) { + this.roleArn = roleArn; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java new file mode 100644 index 000000000..2c634ce70 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java @@ -0,0 +1,680 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.securityanalytics.threatIntel.model; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.UUIDs; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.jobscheduler.spi.ScheduledJobParameter; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.jobscheduler.spi.schedule.Schedule; +import org.opensearch.jobscheduler.spi.schedule.ScheduleParser; +import org.opensearch.securityanalytics.threatIntel.common.RefreshType; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.sacommons.TIFSourceConfig; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; + +/** + * Implementation of TIF Config to store the source configuration metadata and to schedule it onto the job scheduler + */ +public class SATIFSourceConfig implements TIFSourceConfig, Writeable, ScheduledJobParameter { + + private static final Logger log = LogManager.getLogger(SATIFSourceConfig.class); + + /** + * Prefix of indices having threatIntel data + */ + public static final String SOURCE_CONFIG_FIELD = "source_config"; + + public static final String NO_ID = ""; + + public static final Long NO_VERSION = 1L; + public static final String VERSION_FIELD = "version"; + public static final String NAME_FIELD = "name"; + public static final String FORMAT_FIELD = "format"; + public static final String TYPE_FIELD = "type"; + public static final String DESCRIPTION_FIELD = "description"; + public static final String CREATED_BY_USER_FIELD = "created_by_user"; + public static final String CREATED_AT_FIELD = "created_at"; + public static final String SOURCE_FIELD = "source"; + public static final String ENABLED_TIME_FIELD = "enabled_time"; + public static final String ENABLED_FOR_SCAN_FIELD = "enabled_for_scan"; + public static final String LAST_UPDATE_TIME_FIELD = "last_update_time"; + public static final String SCHEDULE_FIELD = "schedule"; + public static final String STATE_FIELD = "state"; + public static final String REFRESH_TYPE_FIELD = "refresh_type"; + public static final String LAST_REFRESHED_TIME_FIELD = "last_refreshed_time"; + public static final String LAST_REFRESHED_USER_FIELD = "last_refreshed_user"; + public static final String ENABLED_FIELD = "enabled"; + public static final String IOC_STORE_FIELD = "ioc_store_config"; + public static final String IOC_TYPES_FIELD = "ioc_types"; + + private String id; + private Long version; + private String name; + private String format; + private SourceConfigType type; + private String description; + private User createdByUser; + private Instant createdAt; + private Source source; + private Instant enabledTime; + private Instant lastUpdateTime; + private Schedule schedule; + private TIFJobState state; + public RefreshType refreshType; + public Instant lastRefreshedTime; + public User lastRefreshedUser; + private Boolean isEnabled; + private IocStoreConfig iocStoreConfig; + private List iocTypes; + private final boolean enabledForScan; + + public SATIFSourceConfig(String id, Long version, String name, String format, SourceConfigType type, String description, User createdByUser, Instant createdAt, Source source, + Instant enabledTime, Instant lastUpdateTime, Schedule schedule, TIFJobState state, RefreshType refreshType, Instant lastRefreshedTime, User lastRefreshedUser, + boolean isEnabled, IocStoreConfig iocStoreConfig, List iocTypes, boolean enabledForScan) { + this.id = id == null ? UUIDs.base64UUID() : id; + this.version = version != null ? version : NO_VERSION; + this.name = name; + this.format = format; + this.type = type; + this.description = description; + this.createdByUser = createdByUser; + this.createdAt = createdAt != null ? createdAt : Instant.now(); + this.source = source; + this.enabledForScan = enabledForScan; + + if (isEnabled && enabledTime == null) { + this.enabledTime = Instant.now(); + } else if (!isEnabled) { + this.enabledTime = null; + } else { + this.enabledTime = enabledTime; + } + + this.lastUpdateTime = lastUpdateTime != null ? lastUpdateTime : Instant.now(); + this.schedule = schedule; + this.state = state != null ? state : TIFJobState.CREATING; + this.refreshType = refreshType != null ? refreshType : RefreshType.FULL; + this.lastRefreshedTime = lastRefreshedTime; + this.lastRefreshedUser = lastRefreshedUser; + this.isEnabled = isEnabled; + this.iocStoreConfig = iocStoreConfig != null ? iocStoreConfig : newIocStoreConfig("default"); + this.iocTypes = iocTypes; + } + + public SATIFSourceConfig(StreamInput sin) throws IOException { + this( + sin.readString(), // id + sin.readLong(), // version + sin.readString(), // name + sin.readString(), // format + sin.readEnum(SourceConfigType.class), // type + sin.readOptionalString(), // description + sin.readBoolean() ? new User(sin) : null, // created by user + sin.readInstant(), // created at + Source.readFrom(sin), // source + sin.readOptionalInstant(), // enabled time + sin.readInstant(), // last update time + sin.readBoolean() ? new IntervalSchedule(sin) : null, // schedule + sin.readEnum(TIFJobState.class), // state + sin.readEnum(RefreshType.class), // refresh type + sin.readOptionalInstant(), // last refreshed time + sin.readBoolean() ? new User(sin) : null, // last refreshed user + sin.readBoolean(), // is enabled + IocStoreConfig.readFrom(sin), // ioc map store + sin.readStringList(), // ioc types + sin.readBoolean() // enabled for scan + ); + } + + public void writeTo(final StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(version); + out.writeString(name); + out.writeString(format); + out.writeEnum(type); + out.writeOptionalString(description); + out.writeBoolean(createdByUser != null); + if (createdByUser != null) { + createdByUser.writeTo(out); + } + out.writeInstant(createdAt); + if (source instanceof S3Source) { + out.writeEnum(Source.Type.S3); + } else if (source instanceof IocUploadSource) { + out.writeEnum(Source.Type.IOC_UPLOAD); + } + source.writeTo(out); + out.writeOptionalInstant(enabledTime); + out.writeInstant(lastUpdateTime); + out.writeBoolean(schedule != null); + if (schedule != null) { + schedule.writeTo(out); + } + out.writeEnum(state); + out.writeEnum(refreshType); + out.writeOptionalInstant(lastRefreshedTime); + out.writeBoolean(lastRefreshedUser != null); + if (lastRefreshedUser != null) { + lastRefreshedUser.writeTo(out); + } + out.writeBoolean(isEnabled); + if (iocStoreConfig instanceof DefaultIocStoreConfig) { + out.writeEnum(IocStoreConfig.Type.DEFAULT); + } + iocStoreConfig.writeTo(out); + out.writeStringCollection(iocTypes); + out.writeBoolean(enabledForScan); + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject() + .startObject(SOURCE_CONFIG_FIELD) + .field(NAME_FIELD, name) + .field(FORMAT_FIELD, format) + .field(TYPE_FIELD, type.name()) + .field(DESCRIPTION_FIELD, description); + + if (createdByUser == null) { + builder.nullField(CREATED_BY_USER_FIELD); + } else { + builder.field(CREATED_BY_USER_FIELD, createdByUser); + } + + if (source == null) { + builder.nullField(SOURCE_FIELD); + } else { + builder.field(SOURCE_FIELD, source); + } + + if (createdAt == null) { + builder.nullField(CREATED_AT_FIELD); + } else { + builder.timeField(CREATED_AT_FIELD, String.format(Locale.getDefault(), "%s_in_millis", CREATED_AT_FIELD), createdAt.toEpochMilli()); + } + + if (enabledTime == null) { + builder.nullField(ENABLED_TIME_FIELD); + } else { + builder.timeField(ENABLED_TIME_FIELD, String.format(Locale.getDefault(), "%s_in_millis", ENABLED_TIME_FIELD), enabledTime.toEpochMilli()); + } + + if (lastUpdateTime == null) { + builder.nullField(LAST_UPDATE_TIME_FIELD); + } else { + builder.timeField(LAST_UPDATE_TIME_FIELD, String.format(Locale.getDefault(), "%s_in_millis", LAST_UPDATE_TIME_FIELD), lastUpdateTime.toEpochMilli()); + } + + if (schedule == null) { + builder.nullField(SCHEDULE_FIELD); + } else { + builder.field(SCHEDULE_FIELD, schedule); + } + + builder.field(STATE_FIELD, state.name()); + builder.field(REFRESH_TYPE_FIELD, refreshType.name()); + if (lastRefreshedTime == null) { + builder.nullField(LAST_REFRESHED_TIME_FIELD); + } else { + builder.timeField(LAST_REFRESHED_TIME_FIELD, String.format(Locale.getDefault(), "%s_in_millis", + LAST_REFRESHED_TIME_FIELD), lastRefreshedTime.toEpochMilli()); + } + if (lastRefreshedUser == null) { + builder.nullField(LAST_REFRESHED_USER_FIELD); + } else { + builder.field(LAST_REFRESHED_USER_FIELD, lastRefreshedUser); + } + builder.field(ENABLED_FIELD, isEnabled); + builder.field(ENABLED_FOR_SCAN_FIELD, enabledForScan); + builder.field(IOC_STORE_FIELD, iocStoreConfig); + builder.field(IOC_TYPES_FIELD, iocTypes); + builder.endObject(); + builder.endObject(); + return builder; + } + + public static SATIFSourceConfig docParse(XContentParser xcp, String id, Long version) throws IOException { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, xcp.nextToken(), xcp); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + SATIFSourceConfig saTifSourceConfig = parse(xcp, id, version); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, xcp.nextToken(), xcp); + + saTifSourceConfig.setId(id); + saTifSourceConfig.setVersion(version); + return saTifSourceConfig; + } + + public static SATIFSourceConfig parse(XContentParser xcp, String id, Long version) throws IOException { + if (id == null) { + id = NO_ID; + } + if (version == null) { + version = NO_VERSION; + } + + String name = null; + String format = null; + SourceConfigType sourceConfigType = null; + String description = null; + User createdByUser = null; + Instant createdAt = null; + Source source = null; + Instant enabledTime = null; + Instant lastUpdateTime = null; + Schedule schedule = null; + TIFJobState state = null; + RefreshType refreshType = null; + Instant lastRefreshedTime = null; + User lastRefreshedUser = null; + boolean isEnabled = true; + boolean enabledForScan = true; + IocStoreConfig iocStoreConfig = null; + List iocTypes = new ArrayList<>(); + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case SOURCE_CONFIG_FIELD: + break; + case NAME_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + name = null; + } else { + name = xcp.text(); + } + break; + case VERSION_FIELD: + version = xcp.longValue(); + break; + case FORMAT_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + format = null; + } else { + format = xcp.text(); + } + break; + case TYPE_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + sourceConfigType = null; + } else { + sourceConfigType = toSourceConfigType(xcp.text()); + } + break; + case DESCRIPTION_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + description = null; + } else { + description = xcp.text(); + } + break; + case CREATED_BY_USER_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + createdByUser = null; + } else { + createdByUser = User.parse(xcp); + } + break; + case CREATED_AT_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + createdAt = null; + } else if (xcp.currentToken().isValue()) { + createdAt = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + createdAt = null; + } + break; + case SOURCE_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + source = null; + } else { + source = Source.parse(xcp); + } + break; + case ENABLED_TIME_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + enabledTime = null; + } else if (xcp.currentToken().isValue()) { + enabledTime = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + enabledTime = null; + } + break; + case LAST_UPDATE_TIME_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + lastUpdateTime = null; + } else if (xcp.currentToken().isValue()) { + lastUpdateTime = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + lastUpdateTime = null; + } + break; + case SCHEDULE_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + schedule = null; + } else { + schedule = ScheduleParser.parse(xcp); + } + break; + case STATE_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + state = TIFJobState.CREATING; + } else { + state = toState(xcp.text()); + } + break; + case REFRESH_TYPE_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + refreshType = null; + } else { + refreshType = toRefreshType(xcp.text()); + } + break; + case LAST_REFRESHED_TIME_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + lastRefreshedTime = null; + } else if (xcp.currentToken().isValue()) { + lastRefreshedTime = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + lastRefreshedTime = null; + } + break; + case LAST_REFRESHED_USER_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + lastRefreshedUser = null; + } else { + lastRefreshedUser = User.parse(xcp); + } + break; + case ENABLED_FIELD: + isEnabled = xcp.booleanValue(); + break; + case ENABLED_FOR_SCAN_FIELD: + enabledForScan = xcp.booleanValue(); + break; + case IOC_STORE_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + iocStoreConfig = null; + } else { + iocStoreConfig = IocStoreConfig.parse(xcp); + } + break; + case IOC_TYPES_FIELD: + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + iocTypes.add(xcp.text()); + } + break; + default: + xcp.skipChildren(); + } + } + + if (isEnabled && enabledTime == null) { + enabledTime = Instant.now(); + } else if (!isEnabled) { + enabledTime = null; + } + + return new SATIFSourceConfig( + id, + version, + name, + format, + sourceConfigType, + description, + createdByUser, + createdAt != null ? createdAt : Instant.now(), + source, + enabledTime, + lastUpdateTime != null ? lastUpdateTime : Instant.now(), + schedule, + state, + refreshType, + lastRefreshedTime, + lastRefreshedUser, + isEnabled, + iocStoreConfig, + iocTypes, + enabledForScan + ); + } + + + public static TIFJobState toState(String stateName) { + try { + return TIFJobState.valueOf(stateName); + } catch (IllegalArgumentException e) { + log.error("Invalid state, cannot be parsed.", e); + return null; + } + } + + public static SourceConfigType toSourceConfigType(String type) { + try { + return SourceConfigType.valueOf(type); + } catch (IllegalArgumentException e) { + log.error("Invalid source config type, cannot be parsed.", e); + return null; + } + } + + public static RefreshType toRefreshType(String stateName) { + try { + return RefreshType.valueOf(stateName); + } catch (IllegalArgumentException e) { + log.error("Invalid refresh type, cannot be parsed.", e); + return null; + } + } + + private IocStoreConfig newIocStoreConfig(String storeType) { + switch (storeType) { + case "default": + return new DefaultIocStoreConfig(new ArrayList<>()); + default: + throw new IllegalStateException("Unexpected store type"); + } + } + + public static SATIFSourceConfig readFrom(StreamInput sin) throws IOException { + return new SATIFSourceConfig(sin); + } + + // Getters and Setters + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public Long getVersion() { + return version; + } + + public void setVersion(Long version) { + this.version = version; + } + + public String getName() { + return this.name; + } + + public void setName(String name) { + this.name = name; + } + + public String getFormat() { + return format; + } + + public void setFormat(String format) { + this.format = format; + } + + public SourceConfigType getType() { + return type; + } + + public void setType(SourceConfigType type) { + this.type = type; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public User getCreatedByUser() { + return createdByUser; + } + + public void setCreatedByUser(User createdByUser) { + this.createdByUser = createdByUser; + } + + public Instant getCreatedAt() { + return createdAt; + } + + public void setCreatedAt(Instant createdAt) { + this.createdAt = createdAt; + } + + public Source getSource() { + return source; + } + + public void setSource(Source source) { + this.source = source; + } + + public Instant getEnabledTime() { + return this.enabledTime; + } + + public void setEnabledTime(Instant enabledTime) { + this.enabledTime = enabledTime; + } + + public Instant getLastUpdateTime() { + return this.lastUpdateTime; + } + + public void setLastUpdateTime(Instant lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; + } + + public Schedule getSchedule() { + return this.schedule; + } + + public void setSchedule(Schedule schedule) { + this.schedule = schedule; + } + + public TIFJobState getState() { + return state; + } + + public void setState(TIFJobState previousState) { + this.state = previousState; + } + + public User getLastRefreshedUser() { + return lastRefreshedUser; + } + + public void setLastRefreshedUser(User lastRefreshedUser) { + this.lastRefreshedUser = lastRefreshedUser; + } + + public Instant getLastRefreshedTime() { + return lastRefreshedTime; + } + + public void setLastRefreshedTime(Instant lastRefreshedTime) { + this.lastRefreshedTime = lastRefreshedTime; + } + + public RefreshType getRefreshType() { + return refreshType; + } + + public void setRefreshType(RefreshType refreshType) { + this.refreshType = refreshType; + } + + public boolean isEnabled() { + return this.isEnabled; + } + + public void enable() { + if (isEnabled == true) { + return; + } + enabledTime = Instant.now(); + isEnabled = true; + } + + public void disable() { + enabledTime = null; + isEnabled = false; + } + + public IocStoreConfig getIocStoreConfig() { + return iocStoreConfig; + } + + public void setIocStoreConfig(IocStoreConfig iocStoreConfig) { + this.iocStoreConfig = iocStoreConfig; + } + + public List getIocTypes() { + return iocTypes; + } + + @Override + public void setIocTypes(List iocTypes) { + this.iocTypes = iocTypes; + } + + @Override + public boolean isEnabledForScan() { + return this.enabledForScan; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java new file mode 100644 index 000000000..222a345ed --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java @@ -0,0 +1,679 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.securityanalytics.threatIntel.model; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchException; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.common.UUIDs; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.jobscheduler.spi.schedule.Schedule; +import org.opensearch.jobscheduler.spi.schedule.ScheduleParser; +import org.opensearch.securityanalytics.model.STIX2IOC; +import org.opensearch.securityanalytics.model.STIX2IOCDto; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; +import org.opensearch.securityanalytics.threatIntel.common.RefreshType; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.sacommons.TIFSourceConfigDto; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.stream.Collectors; + +/** + * Implementation of TIF Config Dto to store the source configuration metadata as DTO object + */ +public class SATIFSourceConfigDto implements Writeable, ToXContentObject, TIFSourceConfigDto { + + private static final Logger log = LogManager.getLogger(SATIFSourceConfigDto.class); + + public static final String SOURCE_CONFIG_FIELD = "source_config"; + + public static final String NO_ID = ""; + + public static final Long NO_VERSION = 1L; + public static final String VERSION_FIELD = "version"; + + public static final String NAME_FIELD = "name"; + public static final String FORMAT_FIELD = "format"; + public static final String TYPE_FIELD = "type"; + + public static final String DESCRIPTION_FIELD = "description"; + public static final String CREATED_BY_USER_FIELD = "created_by_user"; + public static final String CREATED_AT_FIELD = "created_at"; + public static final String SOURCE_FIELD = "source"; + public static final String ENABLED_TIME_FIELD = "enabled_time"; + public static final String ENABLED_FOR_SCAN_FIELD = "enabled_for_scan"; + public static final String LAST_UPDATE_TIME_FIELD = "last_update_time"; + public static final String SCHEDULE_FIELD = "schedule"; + public static final String STATE_FIELD = "state"; + public static final String REFRESH_TYPE_FIELD = "refresh_type"; + public static final String LAST_REFRESHED_TIME_FIELD = "last_refreshed_time"; + public static final String LAST_REFRESHED_USER_FIELD = "last_refreshed_user"; + public static final String ENABLED_FIELD = "enabled"; + public static final String IOC_TYPES_FIELD = "ioc_types"; + + private String id; + private Long version; + private String name; + private String format; + private SourceConfigType type; + private String description; + private User createdByUser; + private Instant createdAt; + private Source source; + private Instant enabledTime; + private Instant lastUpdateTime; + private Schedule schedule; + private TIFJobState state; + public RefreshType refreshType; + public Instant lastRefreshedTime; + public User lastRefreshedUser; + private Boolean isEnabled; + private List iocTypes; + private final boolean enabledForScan; + + public SATIFSourceConfigDto(SATIFSourceConfig saTifSourceConfig) { + this.id = saTifSourceConfig.getId(); + this.version = saTifSourceConfig.getVersion(); + this.name = saTifSourceConfig.getName(); + this.format = saTifSourceConfig.getFormat(); + this.type = saTifSourceConfig.getType(); + this.description = saTifSourceConfig.getDescription(); + this.createdByUser = saTifSourceConfig.getCreatedByUser(); + this.createdAt = saTifSourceConfig.getCreatedAt(); + this.source = saTifSourceConfig.getSource(); + this.enabledTime = saTifSourceConfig.getEnabledTime(); + this.lastUpdateTime = saTifSourceConfig.getLastUpdateTime(); + this.schedule = saTifSourceConfig.getSchedule(); + this.state = saTifSourceConfig.getState(); + this.refreshType = saTifSourceConfig.getRefreshType(); + this.lastRefreshedTime = saTifSourceConfig.getLastRefreshedTime(); + this.lastRefreshedUser = saTifSourceConfig.getLastRefreshedUser(); + this.isEnabled = saTifSourceConfig.isEnabled(); + this.iocTypes = saTifSourceConfig.getIocTypes(); + this.enabledForScan = saTifSourceConfig.isEnabledForScan(); + } + + private List convertToIocDtos(List stix2IocList) { + return stix2IocList.stream() + .map(STIX2IOCDto::new) + .collect(Collectors.toList()); + } + + public SATIFSourceConfigDto(String id, Long version, String name, String format, SourceConfigType type, String description, User createdByUser, Instant createdAt, Source source, + Instant enabledTime, Instant lastUpdateTime, Schedule schedule, TIFJobState state, RefreshType refreshType, Instant lastRefreshedTime, User lastRefreshedUser, + boolean isEnabled, List iocTypes, boolean enabledForScan) { + this.id = id == null ? UUIDs.base64UUID() : id; + this.version = version != null ? version : NO_VERSION; + this.name = name; + this.format = format; + this.type = type; + this.description = description; + this.createdByUser = createdByUser; + this.source = source; + this.createdAt = createdAt != null ? createdAt : Instant.now(); + + if (isEnabled && enabledTime == null) { + this.enabledTime = Instant.now(); + } else if (!isEnabled) { + this.enabledTime = null; + } else { + this.enabledTime = enabledTime; + } + + this.lastUpdateTime = lastUpdateTime != null ? lastUpdateTime : Instant.now(); + this.schedule = schedule; + this.state = state != null ? state : TIFJobState.CREATING; + this.refreshType = refreshType != null ? refreshType : RefreshType.FULL; + this.lastRefreshedTime = lastRefreshedTime; + this.lastRefreshedUser = lastRefreshedUser; + this.isEnabled = isEnabled; + this.iocTypes = iocTypes; + this.enabledForScan = enabledForScan; + } + + public SATIFSourceConfigDto(StreamInput sin) throws IOException { + this( + sin.readString(), // id + sin.readLong(), // version + sin.readString(), // name + sin.readString(), // format + sin.readEnum(SourceConfigType.class), // type + sin.readOptionalString(), // description + sin.readBoolean() ? new User(sin) : null, // created by user + sin.readInstant(), // created at + Source.readFrom(sin), // source + sin.readOptionalInstant(), // enabled time + sin.readInstant(), // last update time + sin.readBoolean() ? new IntervalSchedule(sin) : null, // schedule + sin.readEnum(TIFJobState.class), // state + sin.readEnum(RefreshType.class), // refresh type + sin.readOptionalInstant(), // last refreshed time + sin.readBoolean() ? new User(sin) : null, // last refreshed user + sin.readBoolean(), // is enabled + sin.readStringList(), // ioc types + sin.readBoolean() + ); + } + + public void writeTo(final StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(version); + out.writeString(name); + out.writeString(format); + out.writeEnum(type); + out.writeOptionalString(description); + out.writeBoolean(createdByUser != null); + if (createdByUser != null) { + createdByUser.writeTo(out); + } + out.writeInstant(createdAt); + if (source instanceof S3Source) { + out.writeEnum(Source.Type.S3); + } else if (source instanceof IocUploadSource) { + out.writeEnum(Source.Type.IOC_UPLOAD); + } + source.writeTo(out); + out.writeOptionalInstant(enabledTime); + out.writeInstant(lastUpdateTime); + out.writeBoolean(schedule != null); + if (schedule != null) { + schedule.writeTo(out); + } + out.writeEnum(state); + out.writeEnum(refreshType); + out.writeOptionalInstant(lastRefreshedTime); + out.writeBoolean(lastRefreshedUser != null); + if (lastRefreshedUser != null) { + lastRefreshedUser.writeTo(out); + } + out.writeBoolean(isEnabled); + out.writeStringCollection(iocTypes); + out.writeBoolean(enabledForScan); + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(); + innerXcontent(builder); + builder.endObject(); + return builder; + } + + public XContentBuilder innerXcontent(XContentBuilder builder) throws IOException { + builder.startObject(SOURCE_CONFIG_FIELD); + builder.field(NAME_FIELD, name) + .field(FORMAT_FIELD, format) + .field(TYPE_FIELD, type.name()) + .field(DESCRIPTION_FIELD, description); + if (createdByUser == null) { + builder.nullField(CREATED_BY_USER_FIELD); + } else { + builder.field(CREATED_BY_USER_FIELD, createdByUser); + } + + if (source == null) { + builder.nullField(SOURCE_FIELD); + } else { + builder.field(SOURCE_FIELD, source); + } + + if (createdAt == null) { + builder.nullField(CREATED_AT_FIELD); + } else { + builder.timeField(CREATED_AT_FIELD, String.format(Locale.getDefault(), "%s_in_millis", CREATED_AT_FIELD), createdAt.toEpochMilli()); + } + + if (enabledTime == null) { + builder.nullField(ENABLED_TIME_FIELD); + } else { + builder.timeField(ENABLED_TIME_FIELD, String.format(Locale.getDefault(), "%s_in_millis", ENABLED_TIME_FIELD), enabledTime.toEpochMilli()); + } + + if (lastUpdateTime == null) { + builder.nullField(LAST_UPDATE_TIME_FIELD); + } else { + builder.timeField(LAST_UPDATE_TIME_FIELD, String.format(Locale.getDefault(), "%s_in_millis", LAST_UPDATE_TIME_FIELD), lastUpdateTime.toEpochMilli()); + } + + if (schedule == null) { + builder.nullField(SCHEDULE_FIELD); + } else { + builder.field(SCHEDULE_FIELD, schedule); + } + + builder.field(STATE_FIELD, state.name()); + builder.field(REFRESH_TYPE_FIELD, refreshType.name()); + if (lastRefreshedTime == null) { + builder.nullField(LAST_REFRESHED_TIME_FIELD); + } else { + builder.timeField(LAST_REFRESHED_TIME_FIELD, String.format(Locale.getDefault(), "%s_in_millis", + LAST_REFRESHED_TIME_FIELD), lastRefreshedTime.toEpochMilli()); + } + if (lastRefreshedUser == null) { + builder.nullField(LAST_REFRESHED_USER_FIELD); + } else { + builder.field(LAST_REFRESHED_USER_FIELD, lastRefreshedUser); + } + builder.field(ENABLED_FIELD, isEnabled); + builder.field(ENABLED_FOR_SCAN_FIELD, enabledForScan); + builder.field(IOC_TYPES_FIELD, iocTypes); + builder.endObject(); + return builder; + } + + public static SATIFSourceConfigDto docParse(XContentParser xcp, String id, Long version) throws IOException { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, xcp.nextToken(), xcp); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + SATIFSourceConfigDto saTifSourceConfigDto = parse(xcp, id, version); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, xcp.nextToken(), xcp); + + saTifSourceConfigDto.setId(id); + saTifSourceConfigDto.setVersion(version); + return saTifSourceConfigDto; + } + + public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long version) throws IOException { + if (version == null) { + version = NO_VERSION; + } + + String name = null; + String format = null; + SourceConfigType sourceConfigType = null; + String description = null; + User createdByUser = null; + Instant createdAt = null; + Source source = null; + Instant enabledTime = null; + Instant lastUpdateTime = null; + Schedule schedule = null; + TIFJobState state = null; + RefreshType refreshType = null; + Instant lastRefreshedTime = null; + User lastRefreshedUser = null; + boolean isEnabled = true; + List iocTypes = new ArrayList<>(); + boolean enabledForScan = true; + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + switch (fieldName) { + case SOURCE_CONFIG_FIELD: + break; + case NAME_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + name = null; + } else { + name = xcp.text(); + } + break; + case FORMAT_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + format = null; + } else { + format = xcp.text(); + } + break; + case TYPE_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + sourceConfigType = null; + } else { + sourceConfigType = toSourceConfigType(xcp.text()); + } + break; + case DESCRIPTION_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + description = null; + } else { + description = xcp.text(); + } + break; + case CREATED_BY_USER_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + createdByUser = null; + } else { + createdByUser = User.parse(xcp); + } + break; + case CREATED_AT_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + createdAt = null; + } else if (xcp.currentToken().isValue()) { + createdAt = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + createdAt = null; + } + break; + case SOURCE_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + source = null; + } else { + source = Source.parse(xcp); + } + break; + case ENABLED_TIME_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + enabledTime = null; + } else if (xcp.currentToken().isValue()) { + enabledTime = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + enabledTime = null; + } + break; + case LAST_UPDATE_TIME_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + lastUpdateTime = null; + } else if (xcp.currentToken().isValue()) { + lastUpdateTime = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + lastUpdateTime = null; + } + break; + case SCHEDULE_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + schedule = null; + } else { + schedule = ScheduleParser.parse(xcp); + } + break; + case STATE_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + state = TIFJobState.CREATING; + } else { + state = toState(xcp.text()); + } + break; + case REFRESH_TYPE_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + refreshType = null; + } else { + refreshType = toRefreshType(xcp.text()); + } + break; + case LAST_REFRESHED_TIME_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + lastRefreshedTime = null; + } else if (xcp.currentToken().isValue()) { + lastRefreshedTime = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + lastRefreshedTime = null; + } + break; + case LAST_REFRESHED_USER_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + lastRefreshedUser = null; + } else { + lastRefreshedUser = User.parse(xcp); + } + break; + case ENABLED_FIELD: + isEnabled = xcp.booleanValue(); + break; + case ENABLED_FOR_SCAN_FIELD: + enabledForScan = xcp.booleanValue(); + break; + case IOC_TYPES_FIELD: + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + iocTypes.add(xcp.text()); + } + break; + default: + xcp.skipChildren(); + } + } + + if (isEnabled && enabledTime == null) { + enabledTime = Instant.now(); + } else if (!isEnabled) { + enabledTime = null; + } + + return new SATIFSourceConfigDto( + id, + version, + name, + format, + sourceConfigType, + description, + createdByUser, + createdAt != null ? createdAt : Instant.now(), + source, + enabledTime, + lastUpdateTime != null ? lastUpdateTime : Instant.now(), + schedule, + state, + refreshType, + lastRefreshedTime, + lastRefreshedUser, + isEnabled, + iocTypes, + enabledForScan + ); + } + + // TODO: refactor out to sa commons + public static TIFJobState toState(String stateName) { + try { + return TIFJobState.valueOf(stateName); + } catch (IllegalArgumentException e) { + log.error("Invalid state, cannot be parsed.", e); + throw SecurityAnalyticsException.wrap(new OpenSearchStatusException("Invalid state, cannot be parsed.", RestStatus.BAD_REQUEST)); + } + } + + public static SourceConfigType toSourceConfigType(String type) { + try { + return SourceConfigType.valueOf(type); + } catch (IllegalArgumentException e) { + log.error("Invalid source config type, cannot be parsed.", e); + throw SecurityAnalyticsException.wrap(new OpenSearchStatusException("Invalid source config type, cannot be parsed.", RestStatus.BAD_REQUEST)); + } + } + + public static RefreshType toRefreshType(String stateName) { + try { + return RefreshType.valueOf(stateName); + } catch (IllegalArgumentException e) { + log.error("Invalid refresh type, cannot be parsed.", e); + throw SecurityAnalyticsException.wrap(new OpenSearchStatusException("Invalid refresh type, cannot be parsed.", RestStatus.BAD_REQUEST)); + } + } + + // Getters and Setters + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public Long getVersion() { + return version; + } + + public void setVersion(Long version) { + this.version = version; + } + + public String getName() { + return this.name; + } + + public void setName(String name) { + this.name = name; + } + + public String getFormat() { + return format; + } + + public void setFormat(String format) { + this.format = format; + } + + public SourceConfigType getType() { + return type; + } + + public void setType(SourceConfigType type) { + this.type = type; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public User getCreatedByUser() { + return createdByUser; + } + + public void setCreatedByUser(User createdByUser) { + this.createdByUser = createdByUser; + } + + public Instant getCreatedAt() { + return createdAt; + } + + public void setCreatedAt(Instant createdAt) { + this.createdAt = createdAt; + } + + public Source getSource() { + return source; + } + + public void setSource(Source source) { + this.source = source; + } + + public Instant getEnabledTime() { + return this.enabledTime; + } + + public void setEnabledTime(Instant enabledTime) { + this.enabledTime = enabledTime; + } + + public Instant getLastUpdateTime() { + return this.lastUpdateTime; + } + + public void setLastUpdateTime(Instant lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; + } + + public Schedule getSchedule() { + return this.schedule; + } + + public void setSchedule(Schedule schedule) { + this.schedule = schedule; + } + + public TIFJobState getState() { + return state; + } + + public void setState(TIFJobState previousState) { + this.state = previousState; + } + + public User getLastRefreshedUser() { + return lastRefreshedUser; + } + + public void setLastRefreshedUser(User lastRefreshedUser) { + this.lastRefreshedUser = lastRefreshedUser; + } + + public Instant getLastRefreshedTime() { + return lastRefreshedTime; + } + + public void setLastRefreshedTime(Instant lastRefreshedTime) { + this.lastRefreshedTime = lastRefreshedTime; + } + + public RefreshType getRefreshType() { + return refreshType; + } + + public void setRefreshType(RefreshType refreshType) { + this.refreshType = refreshType; + } + + public boolean isEnabled() { + return this.isEnabled; + } + + /** + * Enable auto update of threat intel feed data + */ + public void enable() { + if (isEnabled == true) { + return; + } + enabledTime = Instant.now(); + isEnabled = true; + } + + /** + * Disable auto update of threat intel feed data + */ + public void disable() { + enabledTime = null; + isEnabled = false; + } + + public List getIocTypes() { + return iocTypes; + } + + public void setIocTypes(List iocTypes) { + this.iocTypes = iocTypes; + } + + public static SATIFSourceConfigDto readFrom(StreamInput sin) throws IOException { + return new SATIFSourceConfigDto(sin); + } + + public boolean isEnabledForScan() { + return enabledForScan; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/Source.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/Source.java new file mode 100644 index 000000000..dcf80a2d9 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/Source.java @@ -0,0 +1,84 @@ +package org.opensearch.securityanalytics.threatIntel.model; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; + +import java.io.IOException; +import java.util.Locale; + +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; + +/** + * Base class for a source object that custom source configs will extend from + */ +public abstract class Source { + private static final Logger log = LogManager.getLogger(Source.class); + abstract String name(); + public static final String S3_FIELD = "s3"; + public static final String IOC_UPLOAD_FIELD = "ioc_upload"; + public static final String URL_DOWNLOAD_FIELD = "url_download"; + + static Source readFrom(StreamInput sin) throws IOException { + Type type = sin.readEnum(Type.class); + switch(type) { + case S3: + return new S3Source(sin); + case IOC_UPLOAD: + return new IocUploadSource(sin); + case URL_DOWNLOAD: + return new UrlDownloadSource(sin); + default: + throw new IllegalStateException("Unexpected input ["+ type + "] when reading ioc store config"); + } + } + + public static Source parse(XContentParser xcp) throws IOException { + Source source = null; + + ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + switch (fieldName) { + case S3_FIELD: + source = S3Source.parse(xcp); + break; + case IOC_UPLOAD_FIELD: + source = IocUploadSource.parse(xcp); + break; + case URL_DOWNLOAD_FIELD: + source = UrlDownloadSource.parse(xcp); + break; + default: + throw new SecurityAnalyticsException( + "Unexpected input in 'source' field when reading ioc store config.", + RestStatus.BAD_REQUEST, + new IllegalArgumentException() + ); + } + } + return source; + } + + public void writeTo(StreamOutput out) throws IOException {} + + enum Type { + S3(), + + IOC_UPLOAD(), + + URL_DOWNLOAD(); + + @Override + public String toString() { + return this.name().toLowerCase(Locale.ROOT); + } + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameter.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFJobParameter.java similarity index 87% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameter.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFJobParameter.java index bcbb84c1c..6e74b3b5a 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameter.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFJobParameter.java @@ -6,7 +6,7 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ -package org.opensearch.securityanalytics.threatIntel.jobscheduler; +package org.opensearch.securityanalytics.threatIntel.model; import org.opensearch.core.ParseField; import org.opensearch.core.common.io.stream.StreamInput; @@ -23,7 +23,6 @@ import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobRequest; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; import java.io.IOException; import java.time.Instant; @@ -52,9 +51,9 @@ public class TIFJobParameter implements Writeable, ScheduledJobParameter { private static final String SCHEDULE_FIELD = "schedule"; private static final String ENABLED_TIME_FIELD = "enabled_time"; private static final String ENABLED_TIME_FIELD_READABLE = "enabled_time_field"; - private static final String state_field = "state"; + private static final String STATE_FIELD = "state"; private static final String INDICES_FIELD = "indices"; - private static final String update_stats_field = "update_stats"; + private static final String UPDATE_STATS_FIELD = "update_stats"; /** @@ -71,9 +70,9 @@ public class TIFJobParameter implements Writeable, ScheduledJobParameter { /** * Additional fields for tif job */ - public static final ParseField STATE_PARSER_FIELD = new ParseField(state_field); + public static final ParseField STATE_PARSER_FIELD = new ParseField(STATE_FIELD); public static final ParseField INDICES_PARSER_FIELD = new ParseField(INDICES_FIELD); - public static final ParseField UPDATE_STATS_PARSER_FIELD = new ParseField(update_stats_field); + public static final ParseField UPDATE_STATS_PARSER_FIELD = new ParseField(UPDATE_STATS_FIELD); /** * Default variables for job scheduling @@ -129,6 +128,71 @@ public class TIFJobParameter implements Writeable, ScheduledJobParameter { */ private UpdateStats updateStats; + public static TIFJobParameter parseFromParser(XContentParser xcp, String id, Long version) throws IOException { + String name = null; + Instant lastUpdateTime = null; + Boolean isEnabled = null; + TIFJobState state = null; + Instant enabledTime = null; + IntervalSchedule schedule = null; + List indices = new ArrayList<>(); + UpdateStats updateStats = null; + + // parsing is coming from the security analytics plugin parser, so it begins with value_string token + XContentParserUtils.ensureExpectedToken(XContentParser.Token.VALUE_STRING, xcp.currentToken(), xcp); + while (true) { + String fieldName = xcp.currentName(); + switch (fieldName) { + case NAME_FIELD: + name = xcp.text(); + break; + case LAST_UPDATE_TIME_FIELD: + lastUpdateTime = Instant.ofEpochMilli(xcp.longValue()); + break; + case ENABLED_TIME_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + enabledTime = null; + } else if (xcp.currentToken().isValue()) { + enabledTime = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + enabledTime = null; + } + break; + case ENABLED_FIELD: + isEnabled = xcp.booleanValue(); + break; + case SCHEDULE_FIELD: + schedule = (IntervalSchedule) ScheduleParser.parse(xcp); + break; + case STATE_FIELD: + state = toState(xcp.text()); + break; + case INDICES_FIELD: + indices = new ArrayList<>(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + indices.add(xcp.text()); + } + break; + case UPDATE_STATS_FIELD: + updateStats = UpdateStats.PARSER.parse(xcp, null); + break; + default: + xcp.skipChildren(); + } + + if (xcp.nextToken() == XContentParser.Token.END_OBJECT){ + break; + } else { + xcp.nextToken(); + } + } + + return new TIFJobParameter(name, lastUpdateTime, enabledTime, isEnabled, schedule, state, indices, updateStats); + } + + // parser used for integ test public static TIFJobParameter parse(XContentParser xcp, String id, Long version) throws IOException { String name = null; Instant lastUpdateTime = null; @@ -151,7 +215,7 @@ public static TIFJobParameter parse(XContentParser xcp, String id, Long version) case ENABLED_FIELD: isEnabled = xcp.booleanValue(); break; - case state_field: + case STATE_FIELD: state = toState(xcp.text()); break; default: @@ -563,7 +627,6 @@ public static TIFJobParameter build(final PutTIFJobRequest request) { ChronoUnit.MINUTES ); return new TIFJobParameter(name, schedule); - } } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFMetadata.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFMetadata.java similarity index 99% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFMetadata.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFMetadata.java index 04486fb7a..20035dcb8 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFMetadata.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFMetadata.java @@ -2,7 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel.common; +package org.opensearch.securityanalytics.threatIntel.model; import java.io.IOException; import java.util.Map; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/UrlDownloadSource.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/UrlDownloadSource.java new file mode 100644 index 000000000..fdc2d9756 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/UrlDownloadSource.java @@ -0,0 +1,117 @@ +package org.opensearch.securityanalytics.threatIntel.model; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.net.URL; + +/** + * This is a Threat Intel Source config where the iocs are downloaded from the URL + */ +public class UrlDownloadSource extends Source implements Writeable, ToXContent { + public static final String URL_FIELD = "url"; + public static final String FEED_FORMAT_FIELD = "feed_format"; + public static final String HAS_CSV_HEADER_FIELD = "has_csv_header_field"; + public static final String CSV_IOC_VALUE_COLUMN_NUM_FIELD = "csv_ioc_value_colum_num"; + public static final String SOURCE_NAME = "URL_DOWNLOAD"; + + private final URL url; + private final String feedFormat; + private final Boolean hasCsvHeader; + private final Integer csvIocValueColumnNo; + + public UrlDownloadSource(URL url, String feedFormat, Boolean hasCsvHeader, Integer csvIocValueColumnNo) { + this.url = url; + this.feedFormat = feedFormat; + this.hasCsvHeader = hasCsvHeader; + this.csvIocValueColumnNo = csvIocValueColumnNo; + + } + + public UrlDownloadSource(StreamInput sin) throws IOException { + this( + new URL(sin.readString()), + sin.readString(), + sin.readOptionalBoolean(), + sin.readOptionalInt() + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(url.toString()); + out.writeString(feedFormat); + out.writeOptionalBoolean(hasCsvHeader); + out.writeOptionalInt(csvIocValueColumnNo); + } + + @Override + String name() { + return SOURCE_NAME; + } + + public URL getUrl() { + return url; + } + + public static UrlDownloadSource parse(XContentParser xcp) throws IOException { + URL url = null; + String feedFormat = null; + Boolean hasCsvHeader = false; + Integer csvIocValueColumnNo = null; + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + switch (fieldName) { + case URL_FIELD: + String urlString = xcp.text(); + url = new URL(urlString); + break; + case FEED_FORMAT_FIELD: + feedFormat = xcp.text(); + break; + case HAS_CSV_HEADER_FIELD: + hasCsvHeader = xcp.booleanValue(); + break; + case CSV_IOC_VALUE_COLUMN_NUM_FIELD: + if (xcp.currentToken() == null) + xcp.skipChildren(); + else + csvIocValueColumnNo = xcp.intValue(); + break; + default: + xcp.skipChildren(); + } + } + return new UrlDownloadSource(url, feedFormat, hasCsvHeader, csvIocValueColumnNo); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .startObject(URL_DOWNLOAD_FIELD) + .field(URL_FIELD, url.toString()) + .field(FEED_FORMAT_FIELD, feedFormat) + .field(HAS_CSV_HEADER_FIELD, hasCsvHeader) + .field(CSV_IOC_VALUE_COLUMN_NUM_FIELD, csvIocValueColumnNo) + .endObject() + .endObject(); + } + + public String getFeedFormat() { + return feedFormat; + } + + public boolean hasCsvHeader() { + return hasCsvHeader; + } + + public Integer getCsvIocValueColumnNo() { + return csvIocValueColumnNo; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/PerIocTypeScanInput.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/PerIocTypeScanInput.java new file mode 100644 index 000000000..902551fd6 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/PerIocTypeScanInput.java @@ -0,0 +1,93 @@ +package org.opensearch.securityanalytics.threatIntel.model.monitor; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class PerIocTypeScanInput implements Writeable, ToXContentObject { + + private static final String IOC_TYPE = "ioc_type"; + private static final String INDEX_TO_FIELDS_MAP = "index_to_fields_map"; + private final String iocType; + private final Map> indexToFieldsMap; + + + public PerIocTypeScanInput(String iocType, Map> indexToFieldsMap) { + this.iocType = iocType; + this.indexToFieldsMap = indexToFieldsMap; + } + + public PerIocTypeScanInput(StreamInput sin) throws IOException { + this( + sin.readString(), + sin.readMapOfLists(StreamInput::readString, StreamInput::readString) + ); + } + + public String getIocType() { + return iocType; + } + + public Map> getIndexToFieldsMap() { + return indexToFieldsMap; + } + + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(iocType); + out.writeMapOfLists(indexToFieldsMap, StreamOutput::writeString, StreamOutput::writeString); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field(IOC_TYPE, iocType) + .field(INDEX_TO_FIELDS_MAP, indexToFieldsMap) + .endObject(); + } + + public static PerIocTypeScanInput parse(XContentParser xcp) throws IOException { + String iocType = ""; + Map> indexToFieldsMap = new HashMap<>(); + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case IOC_TYPE: + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) iocType = xcp.text(); + break; + case INDEX_TO_FIELDS_MAP: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + indexToFieldsMap = null; + } else { + indexToFieldsMap = xcp.map(HashMap::new, p -> { + List fields = new ArrayList<>(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) fields.add(xcp.text()); + } + return fields; + }); + } + break; + default: + xcp.skipChildren(); + } + } + return new PerIocTypeScanInput(iocType, indexToFieldsMap); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInput.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInput.java new file mode 100644 index 000000000..23ca8ca8f --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInput.java @@ -0,0 +1,81 @@ +package org.opensearch.securityanalytics.threatIntel.model.monitor; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class ThreatIntelInput implements Writeable, ToXContentObject { + + public static final String PER_IOC_TYPE_SCAN_INPUTS_FIELD = "per_ioc_type_scan_input_list"; + private final List perIocTypeScanInputList; + + public ThreatIntelInput( + List perIocTypeScanInputList) { + this.perIocTypeScanInputList = perIocTypeScanInputList; + } + + public ThreatIntelInput(StreamInput sin) throws IOException { + this( + sin.readList(PerIocTypeScanInput::new) + ); + } + + public static ThreatIntelInput parse(XContentParser xcp) throws IOException { + List perIocTypeScanInputs = new ArrayList<>(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + switch (fieldName) { + case PER_IOC_TYPE_SCAN_INPUTS_FIELD: + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + PerIocTypeScanInput input = PerIocTypeScanInput.parse(xcp); + perIocTypeScanInputs.add(input); + } + break; + default: + xcp.skipChildren(); + break; + } + } + return new ThreatIntelInput(perIocTypeScanInputs); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeList(perIocTypeScanInputList); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field(PER_IOC_TYPE_SCAN_INPUTS_FIELD, perIocTypeScanInputList) + .endObject(); + } + + public static PerIocTypeScanInput readFrom(StreamInput sin) throws IOException { + return new PerIocTypeScanInput(sin); + } + + public BytesReference getThreatIntelInputAsBytesReference() throws IOException { + BytesStreamOutput out = new BytesStreamOutput(); + this.writeTo(out); + BytesReference bytes = out.bytes(); + return bytes; + } + + public List getPerIocTypeScanInputList() { + return perIocTypeScanInputList; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelTrigger.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelTrigger.java new file mode 100644 index 000000000..a2c35f409 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelTrigger.java @@ -0,0 +1,91 @@ +package org.opensearch.securityanalytics.threatIntel.model.monitor; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class ThreatIntelTrigger implements Writeable, ToXContentObject { + public static final String DATA_SOURCES = "data_sources"; + public static final String IOC_TYPES = "ioc_types"; + List dataSources; + List iocTypes; + + public ThreatIntelTrigger(List dataSources, List iocTypes) { + this.dataSources = dataSources == null ? Collections.emptyList() : dataSources; + this.iocTypes = iocTypes == null ? Collections.emptyList() : iocTypes; + } + + public ThreatIntelTrigger(StreamInput sin) throws IOException { + this( + sin.readStringList(), + sin.readStringList() + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeStringCollection(dataSources); + out.writeStringCollection(iocTypes); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field(DATA_SOURCES, dataSources) + .field(IOC_TYPES, iocTypes) + .endObject(); + } + + public static ThreatIntelTrigger readFrom(StreamInput sin) throws IOException { + return new ThreatIntelTrigger(sin); + } + + public static ThreatIntelTrigger parse(XContentParser xcp) throws IOException { + List iocTypes = new ArrayList<>(); + List dataSources = new ArrayList<>(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case IOC_TYPES: + List vals = new ArrayList<>(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) vals.add(xcp.text()); + } + iocTypes.addAll(vals); + break; + case DATA_SOURCES: + List ds = new ArrayList<>(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) ds.add(xcp.text()); + } + dataSources.addAll(ds); + break; + default: + xcp.skipChildren(); + } + } + return new ThreatIntelTrigger(dataSources, iocTypes); + } + + public List getDataSources() { + return dataSources; + } + + public List getIocTypes() { + return iocTypes; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/TransportThreatIntelMonitorFanOutAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/TransportThreatIntelMonitorFanOutAction.java new file mode 100644 index 000000000..2421e5e5c --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/TransportThreatIntelMonitorFanOutAction.java @@ -0,0 +1,396 @@ +package org.opensearch.securityanalytics.threatIntel.model.monitor; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.commons.alerting.action.DocLevelMonitorFanOutRequest; +import org.opensearch.commons.alerting.action.DocLevelMonitorFanOutResponse; +import org.opensearch.commons.alerting.model.DocumentLevelTriggerRunResult; +import org.opensearch.commons.alerting.model.InputRunResults; +import org.opensearch.commons.alerting.model.Monitor; +import org.opensearch.commons.alerting.model.MonitorRunResult; +import org.opensearch.commons.alerting.model.remote.monitors.RemoteDocLevelMonitorInput; +import org.opensearch.commons.alerting.util.AlertingException; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.seqno.SequenceNumbers; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.sort.SortOrder; +import org.opensearch.securityanalytics.threatIntel.iocscan.dto.IocScanContext; +import org.opensearch.securityanalytics.threatIntel.iocscan.service.SaIoCScanService; +import org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelMonitorRunner; +import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; +import org.opensearch.securityanalytics.util.IndexUtils; +import org.opensearch.tasks.Task; +import org.opensearch.transport.TransportService; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.BiConsumer; + +import static org.opensearch.securityanalytics.threatIntel.util.ThreatIntelMonitorUtils.getThreatIntelInputFromBytesReference; +import static org.opensearch.securityanalytics.util.IndexUtils.getConcreteindexToMonitorInputIndicesMap; + +public class TransportThreatIntelMonitorFanOutAction extends HandledTransportAction { + private static final Logger log = LogManager.getLogger(TransportThreatIntelMonitorFanOutAction.class); + private final ClusterService clusterService; + + private final Settings settings; + private final SATIFSourceConfigService saTifSourceConfigService; + + private final Client client; + + private final NamedXContentRegistry xContentRegistry; + private final SaIoCScanService saIoCScanService; + private final IndexNameExpressionResolver indexNameExpressionResolver; + + @Inject + public TransportThreatIntelMonitorFanOutAction( + TransportService transportService, + Client client, + NamedXContentRegistry xContentRegistry, + ClusterService clusterService, + Settings settings, + ActionFilters actionFilters, + SATIFSourceConfigService saTifSourceConfigService, + SaIoCScanService saIoCScanService, + IndexNameExpressionResolver indexNameExpressionResolver + ) { + super(ThreatIntelMonitorRunner.FAN_OUT_ACTION_NAME, transportService, actionFilters, DocLevelMonitorFanOutRequest::new); + this.clusterService = clusterService; + this.client = client; + this.xContentRegistry = xContentRegistry; + this.settings = settings; + this.saTifSourceConfigService = saTifSourceConfigService; + this.saIoCScanService = saIoCScanService; + this.indexNameExpressionResolver = indexNameExpressionResolver; + } + + @Override + protected void doExecute(Task task, DocLevelMonitorFanOutRequest request, ActionListener actionListener) { + try { + Monitor monitor = request.getMonitor(); + MonitorRunResult monitorResult = new MonitorRunResult<>( + monitor.getName(), + Instant.now(), + Instant.now(), + null, + new InputRunResults(Collections.emptyList(), null, null), + Collections.emptyMap() + ); + + // fetch list of threat intel data containing indices per indicator type + + saTifSourceConfigService.getIocTypeToIndices(ActionListener.wrap( + iocTypeToIndicesMap -> { + onGetIocTypeToIndices(iocTypeToIndicesMap, request, actionListener); + }, e -> { + log.error(() -> new ParameterizedMessage("Unexpected Failure in threat intel monitor {} fan out action while fetching threat intel ioc indices", request.getMonitor().getId()), e); + actionListener.onResponse( + new DocLevelMonitorFanOutResponse( + clusterService.localNode().getId(), + request.getExecutionId(), + request.getMonitor().getId(), + request.getMonitorMetadata().getLastRunContext(), + new InputRunResults(Collections.emptyList(), null, null), + Collections.emptyMap(),//TODO trigger results, + new AlertingException("Fan action of threat intel monitor failed", RestStatus.INTERNAL_SERVER_ERROR, e) + ) + ); + } + )); + + } catch (Exception ex) { + log.error(() -> new ParameterizedMessage("Unexpected Failure in threat intel monitor {} fan out action", request.getMonitor().getId()), ex); + actionListener.onFailure(ex); + } + } + + private void onGetIocTypeToIndices(Map> iocTypeToIndicesMap, DocLevelMonitorFanOutRequest request, ActionListener actionListener) throws IOException { + RemoteDocLevelMonitorInput remoteDocLevelMonitorInput = (RemoteDocLevelMonitorInput) request.getMonitor().getInputs().get(0); + List indices = remoteDocLevelMonitorInput.getDocLevelMonitorInput().getIndices(); + ThreatIntelInput threatIntelInput = getThreatIntelInputFromBytesReference(remoteDocLevelMonitorInput.getInput(), xContentRegistry); + // TODO update fanout request to add mapping of monitor.input.indices' index to concrete index name. + // right now we can't say which one of aliases/index pattern has resolved to this concrete index name + // + // Map> fieldsToFetchPerIndex = new HashMap<>(); alias -> fields mapping is given but we have concrete index name + List fieldsToFetch = new ArrayList<>(); + threatIntelInput.getPerIocTypeScanInputList().forEach(perIocTypeScanInput -> { + perIocTypeScanInput.getIndexToFieldsMap().values().forEach(fieldsToFetch::addAll); +// Map> indexToFieldsMapPerInput = perIocTypeScanInput.getIndexToFieldsMap(); +// for (String index : indexToFieldsMapPerInput.keySet()) { +// List strings = fieldsToFetchPerIndex.computeIfAbsent( +// perIocTypeScanInput.getIocType(), +// k -> new ArrayList<>()); +// strings.addAll(indexToFieldsMapPerInput.get(index)); +// } + }); + + // function passed to update last run context with new max sequence number +// Map updatedLastRunContext = request.getIndexExecutionContext().getUpdatedLastRunContext(); + Map updatedLastRunContext = request.getMonitorMetadata().getLastRunContext(); + BiConsumer lastRunContextUpdateConsumer = (shardId, value) -> { + String indexName = shardId.getIndexName(); + if (updatedLastRunContext.containsKey(indexName)) { + HashMap context = (HashMap) updatedLastRunContext.putIfAbsent(indexName, new HashMap()); + context.put(String.valueOf(shardId.getId()), value); + } else { + log.error("monitor metadata for threat intel monitor {} expected to contain last run context for index {}", + request.getMonitor().getId(), indexName); + } + }; + ActionListener> searchHitsListener = ActionListener.wrap( + (List hits) -> { + if (hits.isEmpty()) { + actionListener.onResponse( + new DocLevelMonitorFanOutResponse( + clusterService.localNode().getId(), + request.getExecutionId(), + request.getMonitor().getId(), + updatedLastRunContext, + new InputRunResults(Collections.emptyList(), null, null), + Collections.emptyMap(), + null + ) + ); + return; + } + BiConsumer resultConsumer = (r, e) -> { + if (e == null) { + actionListener.onResponse( + new DocLevelMonitorFanOutResponse( + clusterService.localNode().getId(), + request.getExecutionId(), + request.getMonitor().getId(), + updatedLastRunContext, + new InputRunResults(Collections.emptyList(), null, null), + Collections.emptyMap(),//TODO trigger results, + null + ) + ); + } else { + actionListener.onFailure(e); + } + }; + Map> concreteindexToMonitorInputIndicesMap = getConcreteindexToMonitorInputIndicesMap( + remoteDocLevelMonitorInput.getDocLevelMonitorInput().getIndices(), + clusterService, + indexNameExpressionResolver); + saIoCScanService.scanIoCs(new IocScanContext<>( + request.getMonitor(), + request.getMonitorMetadata(), + false, + hits, + threatIntelInput, + indices, + iocTypeToIndicesMap, + concreteindexToMonitorInputIndicesMap + ), resultConsumer); + }, + e -> { + log.error("unexpected error while trying to query shards and fetch docs before scanning for malicious IoC's", e); + actionListener.onFailure(e); + } + ); + fetchDataFromShards(request, + fieldsToFetch, + lastRunContextUpdateConsumer, + searchHitsListener); + } + + private void fetchDataFromShards( + DocLevelMonitorFanOutRequest request, + List fieldsToFetch, + BiConsumer updateLastRunContext, + ActionListener> searchHitsListener) { + if (request.getShardIds().isEmpty()) + return; + GroupedActionListener searchHitsFromAllShardsListener = new GroupedActionListener<>( + ActionListener.wrap( + searchHitsOrExceptionCollection -> { + List hits = new ArrayList<>(); + for (SearchHitsOrException searchHitsOrException : searchHitsOrExceptionCollection) { + if (searchHitsOrException.exception == null) { + hits.addAll(searchHitsOrException.hits); + } // else not logging exception as groupedListener onResponse() will log error message + } + searchHitsListener.onResponse(hits); + }, e -> { + log.error("unexpected failure while fetch documents for threat intel monitor " + request.getMonitor().getId(), e); + searchHitsListener.onResponse(Collections.emptyList()); + } + ), request.getShardIds().size() + ); + for (ShardId shardId : request.getShardIds()) { + String shard = shardId.getId() + ""; + Map lastRunContext = request.getMonitorMetadata().getLastRunContext(); + if (lastRunContext.containsKey(shardId.getIndexName()) && lastRunContext.get(shardId.getIndexName()) instanceof Map) { + HashMap shardLastSeenMapForIndex = (HashMap) lastRunContext.get(shardId.getIndexName()); + Long prevSeqNo = shardLastSeenMapForIndex.get(shard) != null ? Long.parseLong(shardLastSeenMapForIndex.get(shard).toString()) : null; + long fromSeqNo = prevSeqNo != null ? prevSeqNo : SequenceNumbers.NO_OPS_PERFORMED; + long toSeqNo = Long.MAX_VALUE; + fetchLatestDocsFromShard(shardId, fromSeqNo, toSeqNo, new ArrayList<>(), request.getMonitor(), shardLastSeenMapForIndex, updateLastRunContext, fieldsToFetch, searchHitsFromAllShardsListener); + } + + } + } + + /** + * recursive function to keep fetching docs in batches of 10000 per search request. all docs with seq_no greater than + * the last seen seq_no are fetched in descending order of sequence number. + */ + + private void fetchLatestDocsFromShard( + ShardId shardId, + long fromSeqNo, long toSeqNo, List searchHitsSoFar, Monitor monitor, + Map shardLastSeenMapForIndex, + BiConsumer updateLastRunContext, + List fieldsToFetch, + GroupedActionListener listener) { + + String shard = shardId.getId() + ""; + try { + if (toSeqNo <= fromSeqNo || toSeqNo < 0) { + listener.onResponse(new SearchHitsOrException(searchHitsSoFar, null)); + return; + } + Long prevSeqNo = shardLastSeenMapForIndex.get(shard) != null ? Long.parseLong(shardLastSeenMapForIndex.get(shard).toString()) : null; + if (toSeqNo > fromSeqNo) { + + searchShard( + shardId.getIndexName(), + shard, + fromSeqNo, + toSeqNo, + Collections.emptyList(), + fieldsToFetch, + ActionListener.wrap( + hits -> { + if (hits.getHits().length == 0) { + if (toSeqNo == Long.MAX_VALUE) { // didn't find any docs + updateLastRunContext.accept(shardId, prevSeqNo != null ? prevSeqNo.toString() : SequenceNumbers.NO_OPS_PERFORMED + ""); + } + listener.onResponse(new SearchHitsOrException(searchHitsSoFar, null)); + return; + } + searchHitsSoFar.addAll(Arrays.asList(hits.getHits())); + if (toSeqNo == Long.MAX_VALUE) { // max sequence number of shard needs to be computed + updateLastRunContext.accept(shardId, String.valueOf(hits.getHits()[0].getSeqNo())); + } + + long leastSeqNoFromHits = hits.getHits()[hits.getHits().length - 1].getSeqNo(); + long updatedToSeqNo = leastSeqNoFromHits - 1; + // recursive call to fetch docs with updated seq no. + fetchLatestDocsFromShard(shardId, fromSeqNo, updatedToSeqNo, searchHitsSoFar, monitor, shardLastSeenMapForIndex, updateLastRunContext, fieldsToFetch, listener); + }, e -> { + if(e.getMessage().contains("all shards failed") && e.getCause().getMessage().contains("No mapping found for [_seq_no] in order to sort on")) { + // this implies that the index being queried doesn't have any docs and hence doesn't understand the in-built _seq_no field mapping + listener.onResponse(new SearchHitsOrException(Collections.emptyList(), null)); + return; + } + log.error(() -> new ParameterizedMessage("Threat intel Monitor {}: Failed to search shard {} in index {}", monitor.getId(), shard, shardId.getIndexName()), e); + listener.onResponse(new SearchHitsOrException(searchHitsSoFar, e)); + } + ) + ); + } + } catch (Exception e) { + log.error(() -> new ParameterizedMessage("threat intel Monitor {}: Failed to run fetch data from shard [{}] of index [{}]", + monitor.getId(), shardId, shardId.getIndexName()), e); + listener.onResponse(new SearchHitsOrException(searchHitsSoFar, e)); + } + } + + public void searchShard( + String index, + String shard, + Long prevSeqNo, + long maxSeqNo, + List docIds, + List fieldsToFetch, + ActionListener listener) { + + if (prevSeqNo != null && prevSeqNo.equals(maxSeqNo) && maxSeqNo != 0L) { + log.debug("Sequence number unchanged."); + listener.onResponse(SearchHits.empty()); + } + + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery() + .filter(QueryBuilders.rangeQuery("_seq_no").gt(prevSeqNo).lte(maxSeqNo)); + + if (docIds != null && !docIds.isEmpty()) { + boolQueryBuilder.filter(QueryBuilders.termsQuery("_id", docIds)); + } + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .version(true) + .sort("_seq_no", SortOrder.DESC) + .seqNoAndPrimaryTerm(true) + .query(boolQueryBuilder) + .size(10000); + + if (!fieldsToFetch.isEmpty()) { + searchSourceBuilder.fetchSource(false); + for (String field : fieldsToFetch) { + searchSourceBuilder.fetchField(field); + } + } + + SearchRequest request = new SearchRequest() + .indices(index) + .preference("_shards:" + shard) + .source(searchSourceBuilder); + + client.search(request, ActionListener.wrap( + response -> { + if (response.status() != RestStatus.OK) { + log.error("Fetching docs from shard failed"); + throw new IOException("Failed to search shard: [" + shard + "] in index [" + index + "]. Response status is " + response.status()); + } + listener.onResponse(response.getHits()); + }, + listener::onFailure // exception logged in invoker method + )); + + } + + public static class SearchHitsOrException { + private final List hits; + private final Exception exception; + + public SearchHitsOrException(List hits, Exception exception) { + assert hits == null || hits.isEmpty() || exception == null; // just a verification that only one of the variables is non-null + this.hits = hits; + this.exception = exception; + } + + public List getHits() { + return hits; + } + + public Exception getException() { + return exception; + } + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestDeleteTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestDeleteTIFSourceConfigAction.java new file mode 100644 index 000000000..a2a8ae49e --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestDeleteTIFSourceConfigAction.java @@ -0,0 +1,50 @@ +package org.opensearch.securityanalytics.threatIntel.resthandler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.node.NodeClient; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.threatIntel.action.SADeleteTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.SADeleteTIFSourceConfigRequest; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +import static org.opensearch.securityanalytics.threatIntel.common.Constants.THREAT_INTEL_SOURCE_CONFIG_ID; + +public class RestDeleteTIFSourceConfigAction extends BaseRestHandler { + + private static final Logger log = LogManager.getLogger(RestDeleteTIFSourceConfigAction.class); + + @Override + public String getName() { + return "delete_tif_config_action"; + } + + @Override + public List routes() { + return List.of(new Route(RestRequest.Method.DELETE, String.format(Locale.getDefault(), "%s/{%s}", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, THREAT_INTEL_SOURCE_CONFIG_ID))); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + String saTifSourceConfigId = request.param(THREAT_INTEL_SOURCE_CONFIG_ID, SATIFSourceConfigDto.NO_ID); + + if (saTifSourceConfigId == null || saTifSourceConfigId.isBlank()) { + throw new IllegalArgumentException("missing id"); + } + + SADeleteTIFSourceConfigRequest req = new SADeleteTIFSourceConfigRequest(saTifSourceConfigId); + + return channel -> client.execute( + SADeleteTIFSourceConfigAction.INSTANCE, + req, + new RestToXContentListener<>(channel) + ); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetIocFindingsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetIocFindingsAction.java new file mode 100644 index 000000000..36927d35d --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetIocFindingsAction.java @@ -0,0 +1,102 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.resthandler; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.action.GetFindingsAction; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsAction; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsRequest; + +import java.io.IOException; +import java.time.DateTimeException; +import java.time.Instant; +import java.util.Arrays; +import java.util.List; + +import static java.util.Collections.singletonList; +import static org.opensearch.rest.RestRequest.Method.GET; + +public class RestGetIocFindingsAction extends BaseRestHandler { + + @Override + public String getName() { + return "get_ioc_findings_action_sa"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + String sortString = request.param("sortString", "timestamp"); + String sortOrder = request.param("sortOrder", "asc"); + String missing = request.param("missing"); + int size = request.paramAsInt("size", 20); + int startIndex = request.paramAsInt("startIndex", 0); + String searchString = request.param("searchString", ""); + + List findingIds = null; + if (request.param("findingIds") != null) { + findingIds = Arrays.asList(request.param("findingIds").split(",")); + } + List iocIds = null; + if (request.param("iocIds") != null) { + iocIds = Arrays.asList(request.param("iocIds").split(",")); + } + Instant startTime = null; + String startTimeParam = request.param("startTime"); + if (startTimeParam != null && !startTimeParam.isEmpty()) { + try { + startTime = Instant.ofEpochMilli(Long.parseLong(startTimeParam)); + } catch (NumberFormatException | NullPointerException | DateTimeException e) { + // Handle the parsing error + // For example, log the error or provide a default value + startTime = Instant.now(); // Default value or fallback + } + } + + Instant endTime = null; + String endTimeParam = request.param("endTime"); + if (endTimeParam != null && !endTimeParam.isEmpty()) { + try { + endTime = Instant.ofEpochMilli(Long.parseLong(endTimeParam)); + } catch (NumberFormatException | NullPointerException | DateTimeException e) { + // Handle the parsing error + // For example, log the error or provide a default value + endTime = Instant.now(); // Default value or fallback + } + } + + Table table = new Table( + sortOrder, + sortString, + missing, + size, + startIndex, + searchString + ); + + GetIocFindingsRequest getIocFindingsRequest = new GetIocFindingsRequest( + findingIds, + iocIds, + startTime, + endTime, + table + ); + return channel -> client.execute( + GetIocFindingsAction.INSTANCE, + getIocFindingsRequest, + new RestToXContentListener<>(channel) + ); + } + + @Override + public List routes() { + return singletonList(new Route(GET, SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings" + "/_search")); + } +} + diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetTIFSourceConfigAction.java new file mode 100644 index 000000000..03ee8a80c --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetTIFSourceConfigAction.java @@ -0,0 +1,52 @@ +package org.opensearch.securityanalytics.threatIntel.resthandler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.node.NodeClient; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestActions; +import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigRequest; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +import static org.opensearch.rest.RestRequest.Method.GET; +import static org.opensearch.securityanalytics.threatIntel.common.Constants.THREAT_INTEL_SOURCE_CONFIG_ID; + +public class RestGetTIFSourceConfigAction extends BaseRestHandler { + + private static final Logger log = LogManager.getLogger(RestGetTIFSourceConfigAction.class); + + @Override + public String getName() { + return "get_tif_config_action"; + } + + @Override + public List routes() { + return List.of(new Route(GET, String.format(Locale.getDefault(), "%s/{%s}", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, THREAT_INTEL_SOURCE_CONFIG_ID))); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + String saTifSourceConfigId = request.param(THREAT_INTEL_SOURCE_CONFIG_ID, SATIFSourceConfigDto.NO_ID); + + if (saTifSourceConfigId == null || saTifSourceConfigId.isEmpty()) { + throw new IllegalArgumentException("missing threat intel source config id"); + } + + SAGetTIFSourceConfigRequest req = new SAGetTIFSourceConfigRequest(saTifSourceConfigId, RestActions.parseVersion(request)); + + return channel -> client.execute( + SAGetTIFSourceConfigAction.INSTANCE, + req, + new RestToXContentListener<>(channel) + ); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestIndexTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestIndexTIFSourceConfigAction.java new file mode 100644 index 000000000..cf3630588 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestIndexTIFSourceConfigAction.java @@ -0,0 +1,81 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.resthandler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.node.NodeClient; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.RestResponse; +import org.opensearch.rest.action.RestResponseListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigRequest; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigResponse; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +import static org.opensearch.securityanalytics.threatIntel.common.Constants.THREAT_INTEL_SOURCE_CONFIG_ID; + +public class RestIndexTIFSourceConfigAction extends BaseRestHandler { + private static final Logger log = LogManager.getLogger(RestIndexTIFSourceConfigAction.class); + @Override + public String getName() { + return "index_tif_config_action"; + } + @Override + public List routes() { + return List.of( + new Route(RestRequest.Method.POST, SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI), + new Route(RestRequest.Method.PUT, String.format(Locale.getDefault(), "%s/{%s}", + SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, THREAT_INTEL_SOURCE_CONFIG_ID)) + ); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + log.debug(String.format(Locale.getDefault(), "%s %s", request.method(), SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI)); + + String id = request.param(THREAT_INTEL_SOURCE_CONFIG_ID, null); + + XContentParser xcp = request.contentParser(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + + SATIFSourceConfigDto tifConfig = SATIFSourceConfigDto.parse(xcp, id, null); + + SAIndexTIFSourceConfigRequest indexTIFConfigRequest = new SAIndexTIFSourceConfigRequest(id, request.method(), tifConfig); + return channel -> client.execute(SAIndexTIFSourceConfigAction.INSTANCE, indexTIFConfigRequest, indexTIFConfigResponse(channel, request.method())); + } + + private RestResponseListener indexTIFConfigResponse(RestChannel channel, RestRequest.Method restMethod) { + return new RestResponseListener<>(channel) { + @Override + public RestResponse buildResponse(SAIndexTIFSourceConfigResponse response) throws Exception { + RestStatus returnStatus = RestStatus.CREATED; + if (restMethod == RestRequest.Method.PUT) { + returnStatus = RestStatus.OK; + } + + BytesRestResponse restResponse = new BytesRestResponse(returnStatus, response.toXContent(channel.newBuilder(), ToXContent.EMPTY_PARAMS)); + + if (restMethod == RestRequest.Method.POST) { + String location = String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, response.getTIFConfigId()); + restResponse.addHeader("Location", location); + } + return restResponse; + } + }; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestListIOCsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestListIOCsAction.java new file mode 100644 index 000000000..e40aa6f71 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestListIOCsAction.java @@ -0,0 +1,74 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.resthandler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.node.NodeClient; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.core.common.Strings; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.RestResponse; +import org.opensearch.rest.action.RestResponseListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsAction; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsActionRequest; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsActionResponse; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +public class RestListIOCsAction extends BaseRestHandler { + private static final Logger log = LogManager.getLogger(RestListIOCsAction.class); + + public String getName() { + return "list_iocs_action"; + } + + public List routes() { + return List.of( + new Route(RestRequest.Method.GET, SecurityAnalyticsPlugin.LIST_IOCS_URI) + ); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + log.debug(String.format(Locale.ROOT, "%s %s", request.method(), SecurityAnalyticsPlugin.LIST_IOCS_URI)); + + // Table params + String sortString = request.param("sortString", "name"); + String sortOrder = request.param("sortOrder", "asc"); + String missing = request.param("missing"); + int size = request.paramAsInt("size", 20); + int startIndex = request.paramAsInt("startIndex", 0); + String searchString = request.param("searchString", ""); + + Table table = new Table( + sortOrder, + sortString, + missing, + size, + startIndex, + searchString + ); + List types = List.of(Strings.commaDelimitedListToStringArray(request.param(ListIOCsActionRequest.TYPE_FIELD, ListIOCsActionRequest.ALL_TYPES_FILTER))); + List feedIds = List.of(Strings.commaDelimitedListToStringArray(request.param(ListIOCsActionRequest.FEED_IDS_FIELD, ""))); + + ListIOCsActionRequest listRequest = new ListIOCsActionRequest(types, feedIds, table); + + return channel -> client.execute(ListIOCsAction.INSTANCE, listRequest, new RestResponseListener<>(channel) { + @Override + public RestResponse buildResponse(ListIOCsActionResponse response) throws Exception { + return new BytesRestResponse(RestStatus.OK, response.toXContent(channel.newBuilder(), ToXContent.EMPTY_PARAMS)); + } + }); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestRefreshTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestRefreshTIFSourceConfigAction.java new file mode 100644 index 000000000..b6c0b1adc --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestRefreshTIFSourceConfigAction.java @@ -0,0 +1,51 @@ +package org.opensearch.securityanalytics.threatIntel.resthandler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.node.NodeClient; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestActions; +import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.threatIntel.action.SARefreshTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.SARefreshTIFSourceConfigRequest; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +import static org.opensearch.securityanalytics.threatIntel.common.Constants.THREAT_INTEL_SOURCE_CONFIG_ID; + +public class RestRefreshTIFSourceConfigAction extends BaseRestHandler { + + private static final Logger log = LogManager.getLogger(RestRefreshTIFSourceConfigAction.class); + + @Override + public String getName() { + return "refresh_tif_config_action"; + } + + @Override + public List routes() { + return List.of(new Route(RestRequest.Method.POST, String.format(Locale.getDefault(), "%s/{%s}/_refresh", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, THREAT_INTEL_SOURCE_CONFIG_ID))); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + String saTifSourceConfigId = request.param(THREAT_INTEL_SOURCE_CONFIG_ID, SATIFSourceConfigDto.NO_ID); + + if (saTifSourceConfigId == null || saTifSourceConfigId.isBlank()) { + throw new IllegalArgumentException("missing id"); + } + + SARefreshTIFSourceConfigRequest req = new SARefreshTIFSourceConfigRequest(saTifSourceConfigId); + + return channel -> client.execute( + SARefreshTIFSourceConfigAction.INSTANCE, + req, + new RestToXContentListener<>(channel) + ); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestSearchTIFSourceConfigsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestSearchTIFSourceConfigsAction.java new file mode 100644 index 000000000..5944bf703 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestSearchTIFSourceConfigsAction.java @@ -0,0 +1,73 @@ +package org.opensearch.securityanalytics.threatIntel.resthandler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.node.NodeClient; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.RestResponse; +import org.opensearch.rest.action.RestResponseListener; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.fetch.subphase.FetchSourceContext; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.threatIntel.action.SASearchTIFSourceConfigsAction; +import org.opensearch.securityanalytics.threatIntel.action.SASearchTIFSourceConfigsRequest; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +import static java.util.Collections.singletonList; +import static org.opensearch.core.rest.RestStatus.OK; +import static org.opensearch.rest.RestRequest.Method.POST; + +public class RestSearchTIFSourceConfigsAction extends BaseRestHandler { + + private static final Logger log = LogManager.getLogger(RestSearchTIFSourceConfigsAction.class); + + @Override + public String getName() { + return "search_tif_configs_action"; + } + + @Override + public List routes() { + return singletonList(new Route(POST, SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/" + "_search")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + log.debug(String.format(Locale.getDefault(), "%s %s", request.method(), SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/" + "_search")); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.parseXContent(request.contentOrSourceParamParser()); + searchSourceBuilder.fetchSource(FetchSourceContext.parseFromRestRequest(request)); + + SASearchTIFSourceConfigsRequest req = new SASearchTIFSourceConfigsRequest(searchSourceBuilder); + + return channel -> client.execute( + SASearchTIFSourceConfigsAction.INSTANCE, + req, + new RestSearchTIFSourceConfigResponseListener(channel, request) + ); + } + + static class RestSearchTIFSourceConfigResponseListener extends RestResponseListener { + private final RestRequest request; + + RestSearchTIFSourceConfigResponseListener(RestChannel channel, RestRequest request) { + super(channel); + this.request = request; + } + + @Override + public RestResponse buildResponse(final SearchResponse response) throws Exception { + return new BytesRestResponse(OK, response.toXContent(channel.newBuilder(), ToXContent.EMPTY_PARAMS)); + } + + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestDeleteThreatIntelMonitorAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestDeleteThreatIntelMonitorAction.java new file mode 100644 index 000000000..362b5955a --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestDeleteThreatIntelMonitorAction.java @@ -0,0 +1,54 @@ +package org.opensearch.securityanalytics.threatIntel.resthandler.monitor; + + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.node.NodeClient; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.threatIntel.action.monitor.DeleteThreatIntelMonitorAction; +import org.opensearch.securityanalytics.threatIntel.action.monitor.request.DeleteThreatIntelMonitorRequest; +import org.opensearch.securityanalytics.threatIntel.action.monitor.request.IndexThreatIntelMonitorRequest; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +import static org.opensearch.securityanalytics.threatIntel.action.monitor.request.IndexThreatIntelMonitorRequest.THREAT_INTEL_MONITOR_ID; + +public class RestDeleteThreatIntelMonitorAction extends BaseRestHandler { + + private static final Logger log = LogManager.getLogger(RestDeleteThreatIntelMonitorAction.class); + + @Override + public String getName() { + return "delete_threat_intel_monitor_action"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + log.debug(String.format(Locale.getDefault(), + "%s %s/{%s}", + request.method(), + SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, + THREAT_INTEL_MONITOR_ID)); + + String detectorId = request.param(THREAT_INTEL_MONITOR_ID); + DeleteThreatIntelMonitorRequest deleteMonitorRequest = new DeleteThreatIntelMonitorRequest(detectorId); + return channel -> client.execute( + DeleteThreatIntelMonitorAction.INSTANCE, + deleteMonitorRequest, new RestToXContentListener<>(channel) + ); + } + + @Override + public List routes() { + return List.of( + new Route(RestRequest.Method.DELETE, String.format(Locale.getDefault(), + "%s/{%s}", + SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, + THREAT_INTEL_MONITOR_ID))); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestGetThreatIntelAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestGetThreatIntelAlertsAction.java new file mode 100644 index 000000000..1cc5266d9 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestGetThreatIntelAlertsAction.java @@ -0,0 +1,89 @@ +package org.opensearch.securityanalytics.threatIntel.resthandler.monitor; + +import java.io.IOException; +import java.time.DateTimeException; +import java.time.Instant; +import java.util.List; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.threatIntel.action.monitor.GetThreatIntelAlertsAction; +import org.opensearch.securityanalytics.threatIntel.action.monitor.request.GetThreatIntelAlertsRequest; + + +import static java.util.Collections.singletonList; +import static org.opensearch.rest.RestRequest.Method.GET; + +public class RestGetThreatIntelAlertsAction extends BaseRestHandler { + + @Override + public String getName() { + return "get_threat_intel_alerts_action"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + + String severityLevel = request.param("severityLevel", "ALL"); + String alertState = request.param("alertState", "ALL"); + // Table params + String sortString = request.param("sortString", "start_time"); + String sortOrder = request.param("sortOrder", "asc"); + String missing = request.param("missing"); + int size = request.paramAsInt("size", 20); + int startIndex = request.paramAsInt("startIndex", 0); + String searchString = request.param("searchString", ""); + + Instant startTime = null; + String startTimeParam = request.param("startTime"); + if (startTimeParam != null && !startTimeParam.isEmpty()) { + try { + startTime = Instant.ofEpochMilli(Long.parseLong(startTimeParam)); + } catch (NumberFormatException | NullPointerException | DateTimeException e) { + startTime = Instant.now(); + } + } + + Instant endTime = null; + String endTimeParam = request.param("endTime"); + if (endTimeParam != null && !endTimeParam.isEmpty()) { + try { + endTime = Instant.ofEpochMilli(Long.parseLong(endTimeParam)); + } catch (NumberFormatException | NullPointerException | DateTimeException e) { + endTime = Instant.now(); + } + } + + Table table = new Table( + sortOrder, + sortString, + missing, + size, + startIndex, + searchString + ); + + GetThreatIntelAlertsRequest req = new GetThreatIntelAlertsRequest( + table, + severityLevel, + alertState, + startTime, + endTime + ); + + return channel -> client.execute( + GetThreatIntelAlertsAction.INSTANCE, + req, + new RestToXContentListener<>(channel) + ); + } + + @Override + public List routes() { + return singletonList(new Route(GET, SecurityAnalyticsPlugin.THREAT_INTEL_ALERTS_URI)); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestIndexThreatIntelMonitorAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestIndexThreatIntelMonitorAction.java new file mode 100644 index 000000000..71485931b --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestIndexThreatIntelMonitorAction.java @@ -0,0 +1,86 @@ +package org.opensearch.securityanalytics.threatIntel.resthandler.monitor; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.node.NodeClient; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.RestResponse; +import org.opensearch.rest.action.RestResponseListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.threatIntel.action.monitor.IndexThreatIntelMonitorAction; +import org.opensearch.securityanalytics.threatIntel.action.monitor.request.IndexThreatIntelMonitorRequest; +import org.opensearch.securityanalytics.threatIntel.action.monitor.response.IndexThreatIntelMonitorResponse; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +public class RestIndexThreatIntelMonitorAction extends BaseRestHandler { + + private static final Logger log = LogManager.getLogger(RestIndexThreatIntelMonitorAction.class); + + @Override + public String getName() { + return "index_threat_intel_monitor_action"; + } + + @Override + public List routes() { + return List.of( + new Route(RestRequest.Method.POST, SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI), + new Route(RestRequest.Method.PUT, String.format(Locale.getDefault(), "%s/{%s}", + SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, IndexThreatIntelMonitorRequest.THREAT_INTEL_MONITOR_ID)) + ); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + log.debug(String.format(Locale.getDefault(), "%s %s", request.method(), SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI)); + + String id = request.param(IndexThreatIntelMonitorRequest.THREAT_INTEL_MONITOR_ID, null); + + XContentParser xcp = request.contentParser(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + + ThreatIntelMonitorDto iocScanMonitor; + try { + iocScanMonitor = ThreatIntelMonitorDto.parse(xcp, id, null); + } catch (Exception e) { + log.error("Failed to parse threat intel monitor: ", e); + throw new SecurityAnalyticsException("Failed to parse threat intel monitor: ", RestStatus.BAD_REQUEST, e); + } + + IndexThreatIntelMonitorRequest indexThreatIntelMonitorRequest = new IndexThreatIntelMonitorRequest(id, request.method(), iocScanMonitor); + return channel -> client.execute(IndexThreatIntelMonitorAction.INSTANCE, indexThreatIntelMonitorRequest, getListener(channel, request.method())); + } + + private RestResponseListener getListener(RestChannel channel, RestRequest.Method restMethod) { + return new RestResponseListener<>(channel) { + @Override + public RestResponse buildResponse(IndexThreatIntelMonitorResponse response) throws Exception { + RestStatus returnStatus = RestStatus.CREATED; + if (restMethod == RestRequest.Method.PUT) { + returnStatus = RestStatus.OK; + } + + BytesRestResponse restResponse = new BytesRestResponse(returnStatus, response.toXContent(channel.newBuilder(), ToXContent.EMPTY_PARAMS)); + + if (restMethod == RestRequest.Method.POST) { + String location = String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, response.getId()); + restResponse.addHeader("Location", location); + } + + return restResponse; + } + }; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestSearchThreatIntelMonitorAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestSearchThreatIntelMonitorAction.java new file mode 100644 index 000000000..047a4f38b --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestSearchThreatIntelMonitorAction.java @@ -0,0 +1,98 @@ +package org.opensearch.securityanalytics.threatIntel.resthandler.monitor; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.node.NodeClient; +import org.opensearch.cluster.routing.Preference; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.RestResponse; +import org.opensearch.rest.action.RestResponseListener; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.fetch.subphase.FetchSourceContext; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.threatIntel.action.monitor.SearchThreatIntelMonitorAction; +import org.opensearch.securityanalytics.threatIntel.action.monitor.request.SearchThreatIntelMonitorRequest; +import org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelMonitorRunner; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +import static org.opensearch.core.rest.RestStatus.OK; +import static org.opensearch.securityanalytics.transport.TransportIndexDetectorAction.PLUGIN_OWNER_FIELD; + +public class RestSearchThreatIntelMonitorAction extends BaseRestHandler { + private static final Logger log = LogManager.getLogger(RestSearchThreatIntelMonitorAction.class); + public static final String SEARCH_THREAT_INTEL_MONITOR_PATH = SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + "_search"; + + @Override + public String getName() { + return "search_threat_intel_monitor_action"; + } + + @Override + public List routes() { + return List.of( + new Route(RestRequest.Method.POST, SEARCH_THREAT_INTEL_MONITOR_PATH)); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + log.debug(String.format(Locale.getDefault(), "%s %s", request.method(), SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + "_search")); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.parseXContent(request.contentOrSourceParamParser()); + searchSourceBuilder.fetchSource(FetchSourceContext.parseFromRestRequest(request)); + searchSourceBuilder.seqNoAndPrimaryTerm(true); + searchSourceBuilder.version(true); + + SearchRequest searchRequest = new SearchRequest(); + searchRequest.source(searchSourceBuilder); + searchRequest.indices(".opendistro-alerting-config");//todo figure out why it should be mentioned here + searchRequest.preference(Preference.PRIMARY_FIRST.type()); + + BoolQueryBuilder boolQueryBuilder; + + if (searchRequest.source().query() == null) { + boolQueryBuilder = new BoolQueryBuilder(); + } else { + boolQueryBuilder = QueryBuilders.boolQuery().must(searchRequest.source().query()); + } + + BoolQueryBuilder bqb = new BoolQueryBuilder(); + bqb.must().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.owner", PLUGIN_OWNER_FIELD))); + bqb.must().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.monitor_type", ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE))); + boolQueryBuilder.filter(bqb); + searchRequest.source().query(boolQueryBuilder); + + SearchThreatIntelMonitorRequest searchThreatIntelMonitorRequest = new SearchThreatIntelMonitorRequest(searchRequest); + + return channel -> { + client.execute(SearchThreatIntelMonitorAction.INSTANCE, searchThreatIntelMonitorRequest, new RestSearchThreatIntelMonitorResponseListener(channel, request)); + }; + } + + static class RestSearchThreatIntelMonitorResponseListener extends RestResponseListener { + private final RestRequest request; + + RestSearchThreatIntelMonitorResponseListener(RestChannel channel, RestRequest request) { + super(channel); + this.request = request; + } + + @Override + public RestResponse buildResponse(final SearchResponse response) throws Exception { + return new BytesRestResponse(OK, response.toXContent(channel.newBuilder(), ToXContent.EMPTY_PARAMS)); + } + + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestUpdateThreatIntelAlertsStatusAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestUpdateThreatIntelAlertsStatusAction.java new file mode 100644 index 000000000..5300dd8f7 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestUpdateThreatIntelAlertsStatusAction.java @@ -0,0 +1,58 @@ +package org.opensearch.securityanalytics.threatIntel.resthandler.monitor; + +import org.apache.commons.lang3.StringUtils; +import org.opensearch.client.node.NodeClient; +import org.opensearch.commons.alerting.model.Alert; +import org.opensearch.core.common.Strings; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.threatIntel.action.monitor.UpdateThreatIntelAlertStatusAction; +import org.opensearch.securityanalytics.threatIntel.action.monitor.request.UpdateThreatIntelAlertStatusRequest; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Locale; + +/** + * Update status of list of threat intel alerts + * Supported state to udpate to : ACKNOWLEDGED, COMPLETED + */ +public class RestUpdateThreatIntelAlertsStatusAction extends BaseRestHandler { + @Override + public String getName() { + return "update_threat_intel_alerts_action"; + } + + @Override + public List routes() { + return Collections.singletonList( + new Route(RestRequest.Method.PUT, String.format( + Locale.getDefault(), + "%s", + SecurityAnalyticsPlugin.THREAT_INTEL_ALERTS_STATUS_URI + ) + )); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + String state = request.param("state"); + if (StringUtils.isBlank(state)) { + throw new IllegalArgumentException("State param is required."); + } + + Alert.State alertState = Alert.State.valueOf(state.toUpperCase()); + List alertIds = List.of( + Strings.commaDelimitedListToStringArray( + request.param(UpdateThreatIntelAlertStatusRequest.ALERT_IDS_FIELD, ""))); + UpdateThreatIntelAlertStatusRequest req = new UpdateThreatIntelAlertStatusRequest(alertIds, alertState); + return channel -> client.execute( + UpdateThreatIntelAlertStatusAction.INSTANCE, + req, + new RestToXContentListener<>(channel) + ); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigAction.java new file mode 100644 index 000000000..8b279d267 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigAction.java @@ -0,0 +1,14 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.sacommons; + +public class IndexTIFSourceConfigAction { + public static final String INDEX_TIF_SOURCE_CONFIG_ACTION_NAME = "cluster:admin/opensearch/securityanalytics/threatintel/sources/write"; + public static final String GET_TIF_SOURCE_CONFIG_ACTION_NAME = "cluster:admin/opensearch/securityanalytics/threatintel/sources/get"; + public static final String DELETE_TIF_SOURCE_CONFIG_ACTION_NAME = "cluster:admin/opensearch/securityanalytics/threatintel/sources/delete"; + public static final String SEARCH_TIF_SOURCE_CONFIGS_ACTION_NAME = "cluster:admin/opensearch/securityanalytics/threatintel/sources/search"; + public static final String REFRESH_TIF_SOURCE_CONFIG_ACTION_NAME = "cluster:admin/opensearch/securityanalytics/threatintel/sources/refresh"; +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigRequest.java new file mode 100644 index 000000000..db33575eb --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigRequest.java @@ -0,0 +1,14 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.sacommons; + +/** + * Threat intel feed config creation request interface + */ +public interface IndexTIFSourceConfigRequest { + String getTIFConfigId(); + TIFSourceConfigDto getTIFConfigDto(); +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigResponse.java new file mode 100644 index 000000000..297efd572 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigResponse.java @@ -0,0 +1,11 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.sacommons; + +public interface IndexTIFSourceConfigResponse { + String getTIFConfigId(); + Long getVersion(); + TIFSourceConfigDto getTIFConfigDto(); +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfig.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfig.java new file mode 100644 index 000000000..dae00034a --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfig.java @@ -0,0 +1,75 @@ +package org.opensearch.securityanalytics.threatIntel.sacommons; + +import org.opensearch.commons.authuser.User; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.jobscheduler.spi.schedule.Schedule; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.model.IocStoreConfig; + +import java.time.Instant; +import java.util.List; + +/** + * Threat intel config interface + */ +public interface TIFSourceConfig { + + public String getId(); + + public void setId(String id); + + Long getVersion(); + + void setVersion(Long version); + + String getName(); + + void setName(String feedName); + + String getFormat(); + + void setFormat(String format); + + SourceConfigType getType(); + + void setType(SourceConfigType type); + + User getCreatedByUser(); + + void setCreatedByUser(User createdByUser); + + Instant getCreatedAt(); + + void setCreatedAt(Instant createdAt); + + Instant getEnabledTime(); + + void setEnabledTime(Instant enabledTime); + + Instant getLastUpdateTime(); + + void setLastUpdateTime(Instant lastUpdateTime); + + Schedule getSchedule(); + + void setSchedule(Schedule schedule); + + TIFJobState getState(); + + void setState(TIFJobState previousState); + + void enable(); + + void disable(); + + IocStoreConfig getIocStoreConfig(); + + void setIocStoreConfig(IocStoreConfig iocStoreConfig); + + public List getIocTypes(); + + public void setIocTypes(List iocTypes); + + boolean isEnabledForScan(); +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigDto.java new file mode 100644 index 000000000..776b0c1b4 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigDto.java @@ -0,0 +1,68 @@ +package org.opensearch.securityanalytics.threatIntel.sacommons; + +import org.opensearch.commons.authuser.User; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.jobscheduler.spi.schedule.Schedule; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; + +import java.time.Instant; +import java.util.List; + +/** + * Threat intel config dto interface + */ +public interface TIFSourceConfigDto { + + public String getId(); + + public void setId(String id); + + Long getVersion(); + + void setVersion(Long version); + + String getName(); + + void setName(String feedName); + + String getFormat(); + + void setFormat(String format); + + SourceConfigType getType(); + + void setType(SourceConfigType type); + + User getCreatedByUser(); + + void setCreatedByUser(User createdByUser); + + Instant getCreatedAt(); + + void setCreatedAt(Instant createdAt); + + Instant getEnabledTime(); + + void setEnabledTime(Instant enabledTime); + + Instant getLastUpdateTime(); + + void setLastUpdateTime(Instant lastUpdateTime); + + Schedule getSchedule(); + + void setSchedule(Schedule schedule); + + TIFJobState getState(); + + void setState(TIFJobState previousState); + + void enable(); + + void disable(); + + public List getIocTypes(); + + public void setIocTypes(List iocTypes); +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigManagementService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigManagementService.java new file mode 100644 index 000000000..9824ff760 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigManagementService.java @@ -0,0 +1,13 @@ +package org.opensearch.securityanalytics.threatIntel.sacommons; + +import org.opensearch.core.action.ActionListener; +public abstract class TIFSourceConfigManagementService { + IndexTIFSourceConfigResponse indexTIFConfig(IndexTIFSourceConfigRequest request, ActionListener listener){ + return null; + } + + // TODO: + // update + // delete + // get +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/IndexIocScanMonitorResponseInterface.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/IndexIocScanMonitorResponseInterface.java new file mode 100644 index 000000000..bf5be489c --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/IndexIocScanMonitorResponseInterface.java @@ -0,0 +1,8 @@ +package org.opensearch.securityanalytics.threatIntel.sacommons.monitor; + +public interface IndexIocScanMonitorResponseInterface { + String getId(); + + ThreatIntelMonitorDto getIocScanMonitor(); +} + diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/IndexTIFSourceConfigRequestInterface.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/IndexTIFSourceConfigRequestInterface.java new file mode 100644 index 000000000..60f233899 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/IndexTIFSourceConfigRequestInterface.java @@ -0,0 +1,4 @@ +package org.opensearch.securityanalytics.threatIntel.sacommons.monitor; + +public interface IndexTIFSourceConfigRequestInterface { +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelAlertDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelAlertDto.java new file mode 100644 index 000000000..4bfc8e502 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelAlertDto.java @@ -0,0 +1,417 @@ +package org.opensearch.securityanalytics.threatIntel.sacommons.monitor; + +import org.opensearch.commons.alerting.model.Alert; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.seqno.SequenceNumbers; +import org.opensearch.securityanalytics.model.threatintel.BaseEntity; +import org.opensearch.securityanalytics.model.threatintel.ThreatIntelAlert; +import org.opensearch.securityanalytics.util.XContentUtils; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.securityanalytics.util.XContentUtils.getInstant; + +public class ThreatIntelAlertDto extends BaseEntity { + + public static final String ALERT_ID_FIELD = "id"; + public static final String SCHEMA_VERSION_FIELD = "schema_version"; + public static final String SEQ_NO_FIELD = "seq_no"; + public static final String PRIMARY_TERM_FIELD = "primary_term"; + public static final String ALERT_VERSION_FIELD = "version"; + public static final String USER_FIELD = "user"; + public static final String TRIGGER_NAME_FIELD = "trigger_id"; + public static final String TRIGGER_ID_FIELD = "trigger_name"; + public static final String STATE_FIELD = "state"; + public static final String START_TIME_FIELD = "start_time"; + public static final String END_TIME_FIELD = "end_time"; + public static final String LAST_UPDATED_TIME_FIELD = "last_updated_time"; + public static final String ACKNOWLEDGED_TIME_FIELD = "acknowledged_time"; + public static final String ERROR_MESSAGE_FIELD = "error_message"; + public static final String SEVERITY_FIELD = "severity"; + public static final String ACTION_EXECUTION_RESULTS_FIELD = "action_execution_results"; + public static final String IOC_VALUE_FIELD = "ioc_value"; + public static final String IOC_TYPE_FIELD = "ioc_type"; + public static final String FINDING_IDS_FIELD = "finding_ids"; + public static final String NO_ID = ""; + public static final long NO_VERSION = 1L; + public static final long NO_SCHEMA_VERSION = 0; + + private final String id; + private final long version; + private final long schemaVersion; + private final long seqNo; + private final long primaryTerm; + private final User user; + private final String triggerName; + private final String triggerId; + private final Alert.State state; + private final Instant startTime; + private final Instant endTime; + private final Instant acknowledgedTime; + private final Instant lastUpdatedTime; + private final String errorMessage; + private final String severity; + private final String iocValue; + private final String iocType; + private List findingIds; + + public ThreatIntelAlertDto( + String id, + long version, + long schemaVersion, + long seqNo, + long primaryTerm, + User user, + String triggerId, + String triggerName, + Alert.State state, + Instant startTime, + Instant endTime, + Instant lastUpdatedTime, + Instant acknowledgedTime, + String errorMessage, + String severity, + String iocValue, + String iocType, + List findingIds + ) { + + this.id = id != null ? id : NO_ID; + this.version = version != 0 ? version : NO_VERSION; + this.schemaVersion = schemaVersion; + this.seqNo = seqNo; + this.primaryTerm = primaryTerm; + this.user = user; + this.triggerId = triggerId; + this.triggerName = triggerName; + this.state = state; + this.startTime = startTime; + this.endTime = endTime; + this.acknowledgedTime = acknowledgedTime; + this.errorMessage = errorMessage; + this.severity = severity; + this.iocValue = iocValue; + this.iocType = iocType; + this.lastUpdatedTime = lastUpdatedTime; + this.findingIds = findingIds; + } + + public ThreatIntelAlertDto(StreamInput sin) throws IOException { + this.id = sin.readString(); + this.version = sin.readLong(); + this.schemaVersion = sin.readLong(); + this.seqNo = sin.readLong(); + this.primaryTerm = sin.readLong(); + this.user = sin.readBoolean() ? new User(sin) : null; + this.triggerId = sin.readString(); + this.triggerName = sin.readString(); + this.state = sin.readEnum(Alert.State.class); + this.startTime = sin.readInstant(); + this.endTime = sin.readOptionalInstant(); + this.acknowledgedTime = sin.readOptionalInstant(); + this.errorMessage = sin.readOptionalString(); + this.severity = sin.readString(); + this.lastUpdatedTime = sin.readOptionalInstant(); + this.iocType = sin.readString(); + this.iocValue = sin.readString(); + this.findingIds = sin.readStringList(); + } + + public ThreatIntelAlertDto(ThreatIntelAlert alert, long seqNo, long primaryTerm) { + this.id = alert.getId(); + this.version = alert.getVersion(); + this.schemaVersion = alert.getSchemaVersion(); + this.user = alert.getUser(); + this.triggerId = alert.getTriggerId(); + this.triggerName = alert.getTriggerName(); + this.state = alert.getState(); + this.startTime = alert.getStartTime(); + this.endTime = alert.getEndTime(); + this.acknowledgedTime = alert.getAcknowledgedTime(); + this.errorMessage = alert.getErrorMessage(); + this.severity = alert.getSeverity(); + this.iocValue = alert.getIocValue(); + this.iocType = alert.getIocType(); + this.lastUpdatedTime = alert.getLastUpdatedTime(); + this.findingIds = alert.getFindingIds(); + this.seqNo = seqNo; + this.primaryTerm = primaryTerm; + } + + public boolean isAcknowledged() { + return state == Alert.State.ACKNOWLEDGED; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(version); + out.writeLong(schemaVersion); + out.writeLong(seqNo); + out.writeLong(primaryTerm); + out.writeBoolean(user != null); + if (user != null) { + user.writeTo(out); + } + out.writeString(triggerId); + out.writeString(triggerName); + out.writeEnum(state); + out.writeInstant(startTime); + out.writeOptionalInstant(endTime); + out.writeOptionalInstant(acknowledgedTime); + out.writeOptionalString(errorMessage); + out.writeString(severity); + out.writeOptionalInstant(lastUpdatedTime); + out.writeString(iocType); + out.writeString(iocValue); + out.writeStringCollection(findingIds); + } + + public static ThreatIntelAlertDto parse(XContentParser xcp, long version) throws IOException { + String id = NO_ID; + long schemaVersion = NO_SCHEMA_VERSION; + long seqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; + long primaryTerm = SequenceNumbers.UNASSIGNED_PRIMARY_TERM; + User user = null; + String triggerId = null; + String triggerName = null; + Alert.State state = null; + Instant startTime = null; + String severity = null; + Instant endTime = null; + Instant acknowledgedTime = null; + Instant lastUpdatedTime = null; + String errorMessage = null; + String iocValue = null; + String iocType = null; + List findingIds = new ArrayList<>(); + + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + switch (fieldName) { + case USER_FIELD: + user = xcp.currentToken() == XContentParser.Token.VALUE_NULL ? null : User.parse(xcp); + break; + case ALERT_ID_FIELD: + id = xcp.text(); + break; + case IOC_VALUE_FIELD: + iocValue = xcp.textOrNull(); + break; + case IOC_TYPE_FIELD: + iocType = xcp.textOrNull(); + break; + case ALERT_VERSION_FIELD: + version = xcp.longValue(); + break; + case SCHEMA_VERSION_FIELD: + schemaVersion = xcp.intValue(); + break; + case SEQ_NO_FIELD: + seqNo = xcp.longValue(); + break; + case PRIMARY_TERM_FIELD: + primaryTerm = xcp.longValue(); + break; + case TRIGGER_ID_FIELD: + triggerId = xcp.text(); + break; + case TRIGGER_NAME_FIELD: + triggerName = xcp.text(); + break; + case STATE_FIELD: + state = Alert.State.valueOf(xcp.text()); + break; + case ERROR_MESSAGE_FIELD: + errorMessage = xcp.textOrNull(); + break; + case SEVERITY_FIELD: + severity = xcp.text(); + break; + case START_TIME_FIELD: + startTime = getInstant(xcp); + break; + case END_TIME_FIELD: + endTime = getInstant(xcp); + break; + case ACKNOWLEDGED_TIME_FIELD: + acknowledgedTime = getInstant(xcp); + break; + case LAST_UPDATED_TIME_FIELD: + lastUpdatedTime = getInstant(xcp); + break; + case FINDING_IDS_FIELD: + ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + findingIds.add(xcp.text()); + } + default: + xcp.skipChildren(); + } + } + + return new ThreatIntelAlertDto(id, + version, + schemaVersion, + seqNo, + primaryTerm, + user, + triggerId, + triggerName, + state, + startTime, + endTime, + acknowledgedTime, + lastUpdatedTime, + errorMessage, + severity, + iocValue, + iocType, + findingIds); + } + + public static Alert readFrom(StreamInput sin) throws IOException { + return new Alert(sin); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return createXContentBuilder(builder, true); + } + + @Override + public String getId() { + return id; + } + + public XContentBuilder toXContentWithUser(XContentBuilder builder) throws IOException { + return createXContentBuilder(builder, false); + } + + private XContentBuilder createXContentBuilder(XContentBuilder builder, boolean secure) throws IOException { + builder.startObject() + .field(ALERT_ID_FIELD, id) + .field(ALERT_VERSION_FIELD, version) + .field(SCHEMA_VERSION_FIELD, schemaVersion) + .field(SEQ_NO_FIELD, seqNo) + .field(PRIMARY_TERM_FIELD, primaryTerm) + .field(TRIGGER_NAME_FIELD, triggerName) + .field(TRIGGER_ID_FIELD, triggerName) + .field(STATE_FIELD, state) + .field(ERROR_MESSAGE_FIELD, errorMessage) + .field(IOC_VALUE_FIELD, iocValue) + .field(IOC_TYPE_FIELD, iocType) + .field(SEVERITY_FIELD, severity) + .field(FINDING_IDS_FIELD, findingIds.toArray(new String[0])); + XContentUtils.buildInstantAsField(builder, acknowledgedTime, ACKNOWLEDGED_TIME_FIELD); + XContentUtils.buildInstantAsField(builder, lastUpdatedTime, LAST_UPDATED_TIME_FIELD); + XContentUtils.buildInstantAsField(builder, startTime, START_TIME_FIELD); + XContentUtils.buildInstantAsField(builder, endTime, END_TIME_FIELD); + if (!secure) { + if (user == null) { + builder.nullField(USER_FIELD); + } else { + builder.field(USER_FIELD, user); + } + } + return builder.endObject(); + } + + public Map asTemplateArg() { + Map map = new HashMap<>(); + map.put(ACKNOWLEDGED_TIME_FIELD, acknowledgedTime != null ? acknowledgedTime.toEpochMilli() : null); + map.put(ALERT_ID_FIELD, id); + map.put(ALERT_VERSION_FIELD, version); + map.put(END_TIME_FIELD, endTime != null ? endTime.toEpochMilli() : null); + map.put(ERROR_MESSAGE_FIELD, errorMessage); + map.put(SEVERITY_FIELD, severity); + map.put(START_TIME_FIELD, startTime.toEpochMilli()); + map.put(STATE_FIELD, state.toString()); + map.put(TRIGGER_ID_FIELD, triggerId); + map.put(TRIGGER_NAME_FIELD, triggerName); + map.put(FINDING_IDS_FIELD, findingIds); + map.put(LAST_UPDATED_TIME_FIELD, lastUpdatedTime); + map.put(IOC_TYPE_FIELD, iocType); + map.put(IOC_VALUE_FIELD, iocValue); + return map; + } + + public long getVersion() { + return version; + } + + public long getSchemaVersion() { + return schemaVersion; + } + + public User getUser() { + return user; + } + + public String getTriggerName() { + return triggerName; + } + + public Alert.State getState() { + return state; + } + + public Instant getStartTime() { + return startTime; + } + + public Instant getEndTime() { + return endTime; + } + + public Instant getAcknowledgedTime() { + return acknowledgedTime; + } + + public String getErrorMessage() { + return errorMessage; + } + + public String getSeverity() { + return severity; + } + + public String getTriggerId() { + return triggerId; + } + + public Instant getLastUpdatedTime() { + return lastUpdatedTime; + } + + public String getIocValue() { + return iocValue; + } + + public String getIocType() { + return iocType; + } + + public List getFindingIds() { + return findingIds; + } + + public long getSeqNo() { + return seqNo; + } + + public long getPrimaryTerm() { + return primaryTerm; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorActions.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorActions.java new file mode 100644 index 000000000..09a4d5fff --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorActions.java @@ -0,0 +1,9 @@ +package org.opensearch.securityanalytics.threatIntel.sacommons.monitor; + +public class ThreatIntelMonitorActions { + public static final String INDEX_THREAT_INTEL_MONITOR_ACTION_NAME = "cluster:admin/opensearch/securityanalytics/threatintel/monitors/write"; + public static final String SEARCH_THREAT_INTEL_MONITOR_ACTION_NAME = "cluster:admin/opensearch/securityanalytics/threatintel/monitors/search"; + public static final String DELETE_THREAT_INTEL_MONITOR_ACTION_NAME = "cluster:admin/opensearch/securityanalytics/threatintel/monitors/delete"; + public static final String GET_THREAT_INTEL_ALERTS_ACTION_NAME = "cluster:admin/opensearch/securityanalytics/threatintel/alerts/get"; + public static final String UPDATE_THREAT_INTEL_ALERT_STATUS_ACTION_NAME = "cluster:admin/opensearch/securityanalytics/threatintel/alerts/status/update"; +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorDto.java new file mode 100644 index 000000000..0070ebddb --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorDto.java @@ -0,0 +1,201 @@ +package org.opensearch.securityanalytics.threatIntel.sacommons.monitor; + +import org.apache.commons.lang3.StringUtils; +import org.opensearch.commons.alerting.model.CronSchedule; +import org.opensearch.commons.alerting.model.Monitor; +import org.opensearch.commons.alerting.model.Schedule; +import org.opensearch.commons.alerting.model.ScheduledJob; +import org.opensearch.commons.alerting.model.remote.monitors.RemoteDocLevelMonitorInput; +import org.opensearch.commons.alerting.model.remote.monitors.RemoteMonitorInput; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.securityanalytics.threatIntel.iocscan.dto.PerIocTypeScanInputDto; +import org.opensearch.securityanalytics.threatIntel.model.monitor.ThreatIntelInput; +import org.opensearch.securityanalytics.threatIntel.util.ThreatIntelMonitorUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +public class ThreatIntelMonitorDto implements Writeable, ToXContentObject, ThreatIntelMonitorDtoInterface { + + private static final String ID = "id"; + public static final String PER_IOC_TYPE_SCAN_INPUT_FIELD = "per_ioc_type_scan_input_list"; + public static final String INDICES = "indices"; + public static final String TRIGGERS_FIELD = "triggers"; + private final String id; + private final String name; + private final List perIocTypeScanInputList; + private final Schedule schedule; + private final boolean enabled; + private final User user; + private final List indices; + private final List triggers; + + public ThreatIntelMonitorDto(String id, String name, List perIocTypeScanInputList, Schedule schedule, boolean enabled, User user, List triggers) { + this.id = StringUtils.isBlank(id) ? UUID.randomUUID().toString() : id; + this.name = name; + this.perIocTypeScanInputList = perIocTypeScanInputList; + this.schedule = schedule; + this.enabled = enabled; + this.user = user; + this.indices = getIndices(perIocTypeScanInputList); + this.triggers = triggers; + } + + private List getIndices(List perIocTypeScanInputList) { + if (perIocTypeScanInputList == null) + return Collections.emptyList(); + List list = new ArrayList<>(); + Set uniqueValues = new HashSet<>(); + for (PerIocTypeScanInputDto dto : perIocTypeScanInputList) { + Map> indexToFieldsMap = dto.getIndexToFieldsMap() == null ? Collections.emptyMap() : dto.getIndexToFieldsMap(); + for (String s : indexToFieldsMap.keySet()) { + if (uniqueValues.add(s)) { + list.add(s); + } + } + } + return list; + } + + public ThreatIntelMonitorDto(StreamInput sin) throws IOException { + this( + sin.readOptionalString(), + sin.readString(), + sin.readList(PerIocTypeScanInputDto::new), + Schedule.readFrom(sin), + sin.readBoolean(), + sin.readBoolean() ? new User(sin) : null, + sin.readList(ThreatIntelTriggerDto::new)); + } + + public static ThreatIntelMonitorDto readFrom(StreamInput sin) throws IOException { + return new ThreatIntelMonitorDto(sin); + } + + public static ThreatIntelMonitorDto parse(XContentParser xcp, String id, Long version) throws IOException { + String name = null; + List inputs = new ArrayList<>(); + Schedule schedule = null; + Boolean enabled = null; + User user = null; + List triggers = new ArrayList<>(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + switch (fieldName) { + case ID: + id = xcp.text(); + break; + case Monitor.NAME_FIELD: + name = xcp.text(); + break; + case PER_IOC_TYPE_SCAN_INPUT_FIELD: + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + PerIocTypeScanInputDto input = PerIocTypeScanInputDto.parse(xcp); + inputs.add(input); + } + break; + case TRIGGERS_FIELD: + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + ThreatIntelTriggerDto input = ThreatIntelTriggerDto.parse(xcp); + triggers.add(input); + } + break; + case Monitor.SCHEDULE_FIELD: + schedule = Schedule.parse(xcp); + break; + case Monitor.ENABLED_FIELD: + enabled = xcp.booleanValue(); + break; + case Monitor.USER_FIELD: + user = xcp.currentToken() == XContentParser.Token.VALUE_NULL ? null : User.parse(xcp); + break; + default: + xcp.skipChildren(); + break; + } + } + + return new ThreatIntelMonitorDto(id, name, inputs, schedule, enabled != null ? enabled : false, user, triggers); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalString(id); + out.writeString(name); + out.writeList(perIocTypeScanInputList); + if (schedule instanceof CronSchedule) { + out.writeEnum(Schedule.TYPE.CRON); + } else { + out.writeEnum(Schedule.TYPE.INTERVAL); + } + schedule.writeTo(out); + out.writeBoolean(enabled); + user.writeTo(out); + out.writeStringCollection(indices); + out.writeList(triggers); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field(ID, id) + .field(Monitor.NAME_FIELD, name) + .field(PER_IOC_TYPE_SCAN_INPUT_FIELD, perIocTypeScanInputList) + .field(Monitor.SCHEDULE_FIELD, schedule) + .field(Monitor.ENABLED_FIELD, enabled) + .field(Monitor.USER_FIELD, user) + .field(INDICES, indices) + .field(TRIGGERS_FIELD, triggers) + .endObject(); + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public List getPerIocTypeScanInputList() { + return perIocTypeScanInputList; + } + + public Schedule getSchedule() { + return schedule; + } + + public boolean isEnabled() { + return enabled; + } + + public User getUser() { + return user; + } + + public List getIndices() { + return indices; + } + + public List getTriggers() { + return triggers; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorDtoInterface.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorDtoInterface.java new file mode 100644 index 000000000..f0cd154cd --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorDtoInterface.java @@ -0,0 +1,4 @@ +package org.opensearch.securityanalytics.threatIntel.sacommons.monitor; + +public interface ThreatIntelMonitorDtoInterface { +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelTriggerDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelTriggerDto.java new file mode 100644 index 000000000..f43512d40 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelTriggerDto.java @@ -0,0 +1,162 @@ +package org.opensearch.securityanalytics.threatIntel.sacommons.monitor; + +import org.apache.commons.lang3.StringUtils; +import org.opensearch.commons.alerting.model.action.Action; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +public class ThreatIntelTriggerDto implements Writeable, ToXContentObject { + + public static final String DATA_SOURCES_FIELD = "data_sources"; + public static final String IOC_TYPES_FIELD = "ioc_types"; + public static final String ACTIONS_FIELD = "actions"; + public static final String ID_FIELD = "id"; + public static final String NAME_FIELD = "name"; + public static final String SEVERITY_FIELD = "severity"; + + private final List dataSources; + private final List iocTypes; + private final List actions; + private final String name; + private final String id; + private final String severity; + + public ThreatIntelTriggerDto(List dataSources, List iocTypes, List actions, String name, String id, String severity) { + this.dataSources = dataSources == null ? Collections.emptyList() : dataSources; + this.iocTypes = iocTypes == null ? Collections.emptyList() : iocTypes; + this.actions = actions; + this.name = name; + this.id = StringUtils.isBlank(id) ? UUID.randomUUID().toString() : id; + this.severity = severity; + } + + public ThreatIntelTriggerDto(StreamInput sin) throws IOException { + this( + sin.readStringList(), + sin.readStringList(), + sin.readList(Action::new), + sin.readString(), + sin.readString(), + sin.readString()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeStringCollection(dataSources); + out.writeStringCollection(iocTypes); + out.writeList(actions); + out.writeString(name); + out.writeString(id); + out.writeString(severity); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field(DATA_SOURCES_FIELD, dataSources) + .field(IOC_TYPES_FIELD, iocTypes) + .field(ACTIONS_FIELD, actions) + .field(ID_FIELD, id) + .field(NAME_FIELD, name) + .field(SEVERITY_FIELD, severity) + .endObject(); + } + + public static ThreatIntelTriggerDto readFrom(StreamInput sin) throws IOException { + return new ThreatIntelTriggerDto(sin); + } + + public static ThreatIntelTriggerDto parse(XContentParser xcp) throws IOException { + List iocTypes = new ArrayList<>(); + List dataSources = new ArrayList<>(); + List actions = new ArrayList<>(); + String name = ""; + String id = null; + String severity = ""; + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case IOC_TYPES_FIELD: + List vals = new ArrayList<>(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) vals.add(xcp.text()); + } + iocTypes.addAll(vals); + break; + case DATA_SOURCES_FIELD: + List ds = new ArrayList<>(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) ds.add(xcp.text()); + } + dataSources.addAll(ds); + break; + case ACTIONS_FIELD: + // Ensure the current token is START_ARRAY, indicating the beginning of the array + XContentParserUtils.ensureExpectedToken( + XContentParser.Token.START_ARRAY, // Expected token type + xcp.currentToken(), // Current token from the parser + xcp // The parser instance + ); + + // Iterate through the array until END_ARRAY token is encountered + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + // Parse each array element into an Action object and add it to the actions list + actions.add(Action.parse(xcp)); + } + break; + case ID_FIELD: + id = xcp.text(); + break; + case NAME_FIELD: + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) name = xcp.text(); + break; + case SEVERITY_FIELD: + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) severity = xcp.text(); + break; + default: + xcp.skipChildren(); + } + } + return new ThreatIntelTriggerDto(dataSources, iocTypes, actions, name, id, severity); + } + + public List getDataSources() { + return dataSources; + } + + public List getIocTypes() { + return iocTypes; + } + + public List getActions() { + return actions; + } + + public String getName() { + return name; + } + + public String getId() { + return id; + } + + public String getSeverity() { + return severity; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DefaultTifSourceConfigLoaderService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DefaultTifSourceConfigLoaderService.java new file mode 100644 index 000000000..c247109d6 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DefaultTifSourceConfigLoaderService.java @@ -0,0 +1,189 @@ +package org.opensearch.securityanalytics.threatIntel.service; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.client.Client; +import org.opensearch.core.action.ActionListener; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.MatchQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.rest.RestRequest; +import org.opensearch.search.SearchHit; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.threatIntel.common.RefreshType; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; +import org.opensearch.transport.RemoteTransportException; + +import java.net.URL; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.opensearch.securityanalytics.util.DetectorUtils.getEmptySearchResponse; + +//todo handle refresh, update tif config +// todo block creation of url based config in transport layer + +public class DefaultTifSourceConfigLoaderService { + private static final Logger log = LogManager.getLogger(DefaultTifSourceConfigLoaderService.class); + private final BuiltInTIFMetadataLoader tifMetadataLoader; + private final Client client; + private final SATIFSourceConfigManagementService satifSourceConfigManagementService; + + public DefaultTifSourceConfigLoaderService(BuiltInTIFMetadataLoader tifMetadataLoader, Client client, SATIFSourceConfigManagementService satifSourceConfigManagementService) { + this.tifMetadataLoader = tifMetadataLoader; + this.client = client; + this.satifSourceConfigManagementService = satifSourceConfigManagementService; + } + + /** + * check if the default tif source configs are loaded. if not, try create them from the feedMetadata.json file. + */ + public void createDefaultTifConfigsIfNotExists(ActionListener listener) { + List tifMetadataList = tifMetadataLoader.getTifMetadataList(); + if (tifMetadataList.isEmpty()) { + log.error("No built-in TIF Configs found"); + listener.onResponse(null); + return; + } + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + for (TIFMetadata tifMetadata : tifMetadataList) { + boolQueryBuilder.should(new MatchQueryBuilder("_id", tifMetadata.getFeedId())); + } + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(boolQueryBuilder).size(9999); + satifSourceConfigManagementService.searchTIFSourceConfigs(searchSourceBuilder, + ActionListener.wrap(searchResponse -> { + createTifConfigsThatDontExist(searchResponse, tifMetadataList, listener); + }, e -> { + if (e instanceof IndexNotFoundException || (e instanceof RemoteTransportException && e.getCause() instanceof IndexNotFoundException)) { + createTifConfigsThatDontExist(getEmptySearchResponse(), tifMetadataList, listener); + } else { + log.error("Failed to search tif config index for default tif configs", e); + listener.onFailure(e); + } + })); + } + + private void createTifConfigsThatDontExist(SearchResponse searchResponse, List tifMetadataList, ActionListener listener) { + Map feedsToCreate = tifMetadataList.stream() + .collect(Collectors.toMap( + TIFMetadata::getFeedId, + Function.identity() + )); + if (searchResponse.getHits() != null && searchResponse.getHits().getHits() != null) { + for (SearchHit hit : searchResponse.getHits().getHits()) { + feedsToCreate.remove(hit.getId()); + } + } + if (feedsToCreate.isEmpty()) { + listener.onResponse(null); + return; + } + GroupedActionListener> groupedActionListener = new GroupedActionListener<>( + new ActionListener<>() { + @Override + public void onResponse(Collection> responseOrExceptions) { + if (responseOrExceptions.stream().allMatch(it -> it.getException() != null)) { // all configs returned error + Exception e = responseOrExceptions.stream().findFirst().get().getException(); + log.error("Failed to create default tif configs", e); + listener.onFailure(e); + return; + } + listener.onResponse(null); + return; + } + + @Override + public void onFailure(Exception e) { + log.error("Unexpected failure while creating Default Threat intel source configs", e); + listener.onFailure(e); + return; + } + }, feedsToCreate.size() + ); + for (TIFMetadata tifMetadata : feedsToCreate.values()) { + if (tifMetadata == null) { + continue; + } + try { + Instant now = Instant.now(); + String iocType = null; + if (tifMetadata.getIocType().equalsIgnoreCase("ip")) { + iocType = IOCType.IPV4_TYPE; + } + satifSourceConfigManagementService.createOrUpdateTifSourceConfig( + new SATIFSourceConfigDto( + tifMetadata.getFeedId(), + SATIFSourceConfigDto.NO_VERSION, + tifMetadata.getName(), + "STIX2", + SourceConfigType.URL_DOWNLOAD, + tifMetadata.getDescription(), + null, + now, + new UrlDownloadSource(new URL(tifMetadata.getUrl()), tifMetadata.getFeedType(), tifMetadata.hasHeader(), tifMetadata.getIocCol()), + now, + now, + new IntervalSchedule(now, 1, ChronoUnit.DAYS), + TIFJobState.CREATING, + RefreshType.FULL, + null, + null, + true, + List.of(iocType), + true + ), + null, + RestRequest.Method.POST, + null, + ActionListener.wrap( + r -> { + groupedActionListener.onResponse(new ResponseOrException<>(r, null)); + }, + e -> { + log.error("failed to create default tif source config " + tifMetadata.getFeedId(), e); + groupedActionListener.onResponse(new ResponseOrException<>(null, e)); + }) + ); + continue; + } catch (Exception ex) { + log.error("Unexpected failure while creating Default Threat intel source configs " + tifMetadata.getFeedId(), ex); + groupedActionListener.onResponse(new ResponseOrException<>(null, ex)); + continue; + } + } + } + + private static class ResponseOrException { + private final R response; + private final Exception exception; + + private ResponseOrException(R response, Exception exception) { + this.response = response; + this.exception = exception; + } + + public R getResponse() { + return response; + } + + public Exception getException() { + return exception; + } + } +} + diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DetectorThreatIntelService.java similarity index 99% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/service/DetectorThreatIntelService.java index e541ee36c..6619b33f5 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DetectorThreatIntelService.java @@ -2,7 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel; +package org.opensearch.securityanalytics.threatIntel.service; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java new file mode 100644 index 000000000..815729f40 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java @@ -0,0 +1,830 @@ +package org.opensearch.securityanalytics.threatIntel.service; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchException; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.delete.DeleteResponse; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexAbstraction; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.jobscheduler.spi.LockModel; +import org.opensearch.rest.RestRequest; +import org.opensearch.search.SearchHit; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.model.STIX2IOC; +import org.opensearch.securityanalytics.model.STIX2IOCDto; +import org.opensearch.securityanalytics.services.STIX2IOCFetchService; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; +import org.opensearch.securityanalytics.threatIntel.model.IocStoreConfig; +import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.stream.Collectors; + +import static org.opensearch.securityanalytics.threatIntel.common.SourceConfigType.IOC_UPLOAD; +import static org.opensearch.securityanalytics.threatIntel.common.SourceConfigType.URL_DOWNLOAD; + +/** + * Service class for threat intel feed source config object + */ +public class SATIFSourceConfigManagementService { + private static final Logger log = LogManager.getLogger(SATIFSourceConfigManagementService.class); + private final SATIFSourceConfigService saTifSourceConfigService; + private final TIFLockService lockService; //TODO: change to js impl lock + private final STIX2IOCFetchService stix2IOCFetchService; + private final NamedXContentRegistry xContentRegistry; + private final ClusterService clusterService; + + /** + * Default constructor + * + * @param saTifSourceConfigService the tif source config dao + * @param lockService the lock service + * @param stix2IOCFetchService the service to download, and store IOCs + */ + @Inject + public SATIFSourceConfigManagementService( + final SATIFSourceConfigService saTifSourceConfigService, + final TIFLockService lockService, + final STIX2IOCFetchService stix2IOCFetchService, + final NamedXContentRegistry xContentRegistry, + final ClusterService clusterService + ) { + this.saTifSourceConfigService = saTifSourceConfigService; + this.lockService = lockService; + this.stix2IOCFetchService = stix2IOCFetchService; + this.xContentRegistry = xContentRegistry; + this.clusterService = clusterService; + } + + public void createOrUpdateTifSourceConfig( + final SATIFSourceConfigDto saTifSourceConfigDto, + final LockModel lock, + final RestRequest.Method restMethod, + final User user, + final ActionListener listener + ) { + if (restMethod == RestRequest.Method.POST) { + createIocAndTIFSourceConfig(saTifSourceConfigDto, lock, user, listener); + } else if (restMethod == RestRequest.Method.PUT) { + updateIocAndTIFSourceConfig(saTifSourceConfigDto, lock, user, listener); + } + } + + /** + * Creates the job index if it doesn't exist and indexes the tif source config object + * + * @param saTifSourceConfigDto the tif source config dto + * @param lock the lock object + * @param listener listener that accepts a tif source config if successful + */ + public void createIocAndTIFSourceConfig( + final SATIFSourceConfigDto saTifSourceConfigDto, + final LockModel lock, + final User createdByUser, + final ActionListener listener + ) { + try { + SATIFSourceConfig saTifSourceConfig = convertToSATIFConfig(saTifSourceConfigDto, null, TIFJobState.CREATING, createdByUser); + + // Don't index iocs into source config index + List iocs; + if (saTifSourceConfig.getSource() instanceof IocUploadSource) { + List iocDtos = ((IocUploadSource) saTifSourceConfigDto.getSource()).getIocs(); + ((IocUploadSource) saTifSourceConfig.getSource()).setIocs(List.of()); + iocs = convertToIocs(iocDtos, saTifSourceConfig.getName(), saTifSourceConfig.getId()); + } else { + iocs = null; + } + + // Index threat intel source config as creating and update the last refreshed time + saTifSourceConfig.setLastRefreshedTime(Instant.now()); + saTifSourceConfig.setLastRefreshedUser(createdByUser); + + saTifSourceConfigService.indexTIFSourceConfig( + saTifSourceConfig, + lock, + ActionListener.wrap( + indexSaTifSourceConfigResponse -> { + log.debug("Indexed threat intel source config as CREATING for [{}]", indexSaTifSourceConfigResponse.getId()); + // Call to download and save IOCS's, update state as AVAILABLE on success + downloadAndSaveIOCs( + indexSaTifSourceConfigResponse, + iocs, + ActionListener.wrap( + r -> { + markSourceConfigAsAction( + indexSaTifSourceConfigResponse, + TIFJobState.AVAILABLE, + ActionListener.wrap( + updateSaTifSourceConfigResponse -> { + log.debug("Updated threat intel source config as AVAILABLE for [{}]", indexSaTifSourceConfigResponse.getId()); + SATIFSourceConfigDto returnedSaTifSourceConfigDto = new SATIFSourceConfigDto(updateSaTifSourceConfigResponse); + listener.onResponse(returnedSaTifSourceConfigDto); + }, e -> { + log.error("Failed to index threat intel source config with id [{}]", indexSaTifSourceConfigResponse.getId()); + listener.onFailure(e); + } + )); + }, + e -> { + log.error("Failed to download and save IOCs for threat intel source config [{}]", indexSaTifSourceConfigResponse.getId(), e); + // set isDeleted as true because we want to delete failed source configs regardless if threat intel monitor exists + deleteAllIocsAndSourceConfig(ActionListener.wrap( + deleteResponse -> { + log.debug("Successfully deleted threat intel source config [{}]", indexSaTifSourceConfigResponse.getId()); + listener.onFailure(e); + }, ex -> { + log.error("Failed to delete threat intel source config [{}]", indexSaTifSourceConfigResponse.getId(), ex); + listener.onFailure(ex); + } + ), indexSaTifSourceConfigResponse, true); + }) + ); + }, e -> { + log.error("Failed to index threat intel source config with id [{}]", saTifSourceConfig.getId()); + listener.onFailure(e); + })); + } catch (Exception e) { + log.error("Failed to create IOCs and threat intel source config"); + listener.onFailure(e); + } + } + + /** + * Function to download and save IOCs, if source is not null, grab IOCs from S3 otherwise IOCs are passed in + * + * @param saTifSourceConfig + * @param stix2IOCList + * @param actionListener + */ + public void downloadAndSaveIOCs(SATIFSourceConfig saTifSourceConfig, + List stix2IOCList, + ActionListener actionListener) { + switch (saTifSourceConfig.getType()) { + case S3_CUSTOM: + stix2IOCFetchService.downloadAndIndexIOCs(saTifSourceConfig, actionListener); + break; + case URL_DOWNLOAD: + stix2IOCFetchService.downloadFromUrlAndIndexIOCs(saTifSourceConfig, actionListener); + break; + case IOC_UPLOAD: + List validStix2IocList = new ArrayList<>(); + // If the IOC received is not a type listed for the config, do not add it to the queue + for (STIX2IOC stix2IOC : stix2IOCList) { + if (saTifSourceConfig.getIocTypes().contains(stix2IOC.getType().toString())) { + validStix2IocList.add(stix2IOC); + } else { + log.error("{} is not a supported Ioc type for threat intel source config {}. Skipping IOC {}: of type {} value {}", + stix2IOC.getType().toString(), saTifSourceConfig.getId(), + stix2IOC.getId(), stix2IOC.getType().toString(), stix2IOC.getValue() + ); + } + } + if (validStix2IocList.isEmpty()) { + log.error("No supported IOCs to index"); + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("No compatible Iocs were uploaded for threat intel source config " + saTifSourceConfig.getName(), RestStatus.BAD_REQUEST))); + return; + } + stix2IOCFetchService.onlyIndexIocs(saTifSourceConfig, validStix2IocList, actionListener); + break; + } + } + + public void getTIFSourceConfig( + final String saTifSourceConfigId, + final ActionListener listener + ) { + saTifSourceConfigService.getTIFSourceConfig(saTifSourceConfigId, ActionListener.wrap( + saTifSourceConfigResponse -> { + SATIFSourceConfigDto returnedSaTifSourceConfigDto = new SATIFSourceConfigDto(saTifSourceConfigResponse); + listener.onResponse(returnedSaTifSourceConfigDto); + }, e -> { + log.error("Failed to get threat intel source config for [{}]", saTifSourceConfigId); + listener.onFailure(e); + } + )); + } + + public void searchTIFSourceConfigs( + final SearchSourceBuilder searchSourceBuilder, + final ActionListener listener + ) { + try { + // convert search response to threat intel source config dtos + saTifSourceConfigService.searchTIFSourceConfigs(searchSourceBuilder, ActionListener.wrap( + searchResponse -> { + for (SearchHit hit : searchResponse.getHits()) { + XContentParser xcp = XContentType.JSON.xContent().createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString() + ); + SATIFSourceConfigDto satifSourceConfigDto = SATIFSourceConfigDto.docParse(xcp, hit.getId(), hit.getVersion()); + XContentBuilder xcb = satifSourceConfigDto.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS); + hit.sourceRef(BytesReference.bytes(xcb)); + } + listener.onResponse(searchResponse); + }, e -> { + log.error("Failed to fetch all threat intel source configs for search request [{}]", searchSourceBuilder, e); + listener.onFailure(e); + } + )); + } catch (Exception e) { + log.error("Failed to search and parse all threat intel source configs"); + listener.onFailure(e); + } + } + + public void updateIocAndTIFSourceConfig( + final SATIFSourceConfigDto saTifSourceConfigDto, + final LockModel lock, + final User updatedByUser, + final ActionListener listener + ) { + try { + saTifSourceConfigService.getTIFSourceConfig(saTifSourceConfigDto.getId(), ActionListener.wrap( + retrievedSaTifSourceConfig -> { + // Due to the lack of a different API to do activate/deactivate we will check if enabled_for_scan variable is changed between model and request. + // If yes, we will ONLY update enabled_for_scan field and ignore any updates to the rest of the fields to simulate a dedicated activate/deactivate API. + if (retrievedSaTifSourceConfig.isEnabledForScan() != saTifSourceConfigDto.isEnabledForScan()) { + // FIXME add a disable_refresh api independent of update api so that it can be supported for default configs also + boolean isEnabled = URL_DOWNLOAD.equals(retrievedSaTifSourceConfig.getType()) ? + saTifSourceConfigDto.isEnabledForScan() : + retrievedSaTifSourceConfig.isEnabled(); + SATIFSourceConfig config = new SATIFSourceConfig( + retrievedSaTifSourceConfig.getId(), + retrievedSaTifSourceConfig.getVersion(), + retrievedSaTifSourceConfig.getName(), + retrievedSaTifSourceConfig.getFormat(), + retrievedSaTifSourceConfig.getType(), + retrievedSaTifSourceConfig.getDescription(), + retrievedSaTifSourceConfig.getCreatedByUser(), + retrievedSaTifSourceConfig.getCreatedAt(), + retrievedSaTifSourceConfig.getSource(), + retrievedSaTifSourceConfig.getEnabledTime(), + retrievedSaTifSourceConfig.getLastUpdateTime(), + retrievedSaTifSourceConfig.getSchedule(), + retrievedSaTifSourceConfig.getState(), + retrievedSaTifSourceConfig.getRefreshType(), + Instant.now(), + updatedByUser, + isEnabled, + retrievedSaTifSourceConfig.getIocStoreConfig(), + retrievedSaTifSourceConfig.getIocTypes(), + saTifSourceConfigDto.isEnabledForScan() // update only enabled_for_scan + ); + internalUpdateTIFSourceConfig(config, ActionListener.wrap( + r -> { + listener.onResponse(new SATIFSourceConfigDto(r)); + }, e -> { + String action = saTifSourceConfigDto.isEnabledForScan() ? "activate" : "deactivate"; + log.error(String.format("Failed to %s tif source config %s", action, retrievedSaTifSourceConfig.getId()), e); + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchException(String.format("Failed to %s tif source config %s", action, retrievedSaTifSourceConfig.getId()), e))); + return; + } + )); + return; + } else if (SourceConfigType.URL_DOWNLOAD.equals(saTifSourceConfigDto.getType()) || saTifSourceConfigDto.getSource() instanceof UrlDownloadSource) { // fail if enabled_for_scan isn't changed and type is url download + log.error("Unsupported Threat intel Source Config Type passed - " + saTifSourceConfigDto.getType()); + listener.onFailure(new UnsupportedOperationException("Unsupported Threat intel Source Config Type passed - " + saTifSourceConfigDto.getType())); + return; + } + + if (TIFJobState.AVAILABLE.equals(retrievedSaTifSourceConfig.getState()) == false && TIFJobState.REFRESH_FAILED.equals(retrievedSaTifSourceConfig.getState()) == false) { + log.error("Invalid threat intel source config state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, retrievedSaTifSourceConfig.getState()); + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException( + String.format(Locale.getDefault(), "Invalid threat intel source config state. Expecting %s or %s but received %s", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, retrievedSaTifSourceConfig.getState()), + RestStatus.BAD_REQUEST))); + return; + } + + if (false == saTifSourceConfigDto.getType().equals(retrievedSaTifSourceConfig.getType())) { + log.error("Unable to update threat intel source config, type cannot change from {} to {}", retrievedSaTifSourceConfig.getType(), saTifSourceConfigDto.getType()); + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException( + String.format(Locale.getDefault(), "Unable to update threat intel source config, type cannot change from %s to %s", retrievedSaTifSourceConfig.getType(), saTifSourceConfigDto.getType()), + RestStatus.BAD_REQUEST))); + return; + } + + SATIFSourceConfig updatedSaTifSourceConfig = updateSaTifSourceConfig(saTifSourceConfigDto, retrievedSaTifSourceConfig); + + // Don't index iocs into source config index + List iocs; + if (updatedSaTifSourceConfig.getSource() instanceof IocUploadSource) { + List iocDtos = ((IocUploadSource) saTifSourceConfigDto.getSource()).getIocs(); + ((IocUploadSource) updatedSaTifSourceConfig.getSource()).setIocs(List.of()); + iocs = convertToIocs(iocDtos, updatedSaTifSourceConfig.getName(), updatedSaTifSourceConfig.getId()); + } else { + iocs = null; + } + + // Download and save IOCS's based on new threat intel source config + updatedSaTifSourceConfig.setLastRefreshedTime(Instant.now()); + updatedSaTifSourceConfig.setLastRefreshedUser(updatedByUser); + markSourceConfigAsAction(updatedSaTifSourceConfig, TIFJobState.REFRESHING, ActionListener.wrap( + r -> { + log.info("Set threat intel source config as REFRESHING for [{}]", updatedSaTifSourceConfig.getId()); + switch (updatedSaTifSourceConfig.getType()) { + case S3_CUSTOM: + downloadAndSaveIocsToRefresh(listener, updatedSaTifSourceConfig, null); + break; + case IOC_UPLOAD: + downloadAndSaveIocsToRefresh(listener, updatedSaTifSourceConfig, iocs); + break; + } + }, e -> { + log.error("Failed to set threat intel source config as REFRESH_FAILED for [{}]", updatedSaTifSourceConfig.getId()); + listener.onFailure(e); + } + )); + }, e -> { + log.error("Failed to get threat intel source config for [{}]", saTifSourceConfigDto.getId()); + listener.onFailure(e); + } + )); + } catch (Exception e) { + log.error("Failed to update IOCs and threat intel source config for [{}]", saTifSourceConfigDto.getId()); + listener.onFailure(e); + } + } + + public void internalUpdateTIFSourceConfig( + final SATIFSourceConfig saTifSourceConfig, + final ActionListener listener + ) { + try { + saTifSourceConfig.setLastUpdateTime(Instant.now()); + saTifSourceConfigService.updateTIFSourceConfig(saTifSourceConfig, listener); + } catch (Exception e) { + log.error("Failed to update threat intel source config [{}]", saTifSourceConfig.getId()); + listener.onFailure(e); + } + } + + public void refreshTIFSourceConfig( + final String saTifSourceConfigId, + final User user, + final ActionListener listener + ) { + saTifSourceConfigService.getTIFSourceConfig(saTifSourceConfigId, ActionListener.wrap( + saTifSourceConfig -> { + if (saTifSourceConfig.getType() == IOC_UPLOAD) { + log.error("Unable to refresh threat intel source config [{}] with a source type of [{}]", saTifSourceConfig.getId(), IOC_UPLOAD); + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException( + String.format(Locale.getDefault(), "Unable to refresh threat intel source config [%s] with a source type of [%s]", saTifSourceConfig.getId(), IOC_UPLOAD), + RestStatus.BAD_REQUEST))); + return; + } + + if (TIFJobState.AVAILABLE.equals(saTifSourceConfig.getState()) == false && TIFJobState.REFRESH_FAILED.equals(saTifSourceConfig.getState()) == false) { + log.error("Invalid threat intel source config state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, saTifSourceConfig.getState()); + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException( + String.format(Locale.getDefault(), "Invalid threat intel source config state. Expecting %s or %s but received %s", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, saTifSourceConfig.getState()), + RestStatus.BAD_REQUEST))); + return; + } + + // set the last refreshed user + if (user != null) { + saTifSourceConfig.setLastRefreshedUser(user); + } + + // REFRESH FLOW + log.debug("Refreshing IOCs and updating threat intel source config"); + saTifSourceConfig.setLastRefreshedTime(Instant.now()); + markSourceConfigAsAction(saTifSourceConfig, TIFJobState.REFRESHING, ActionListener.wrap( + updatedSourceConfig -> { + downloadAndSaveIocsToRefresh(listener, updatedSourceConfig, null); + }, e -> { + log.error("Failed to set threat intel source config as REFRESHING for [{}]", saTifSourceConfig.getId()); + listener.onFailure(e); + } + )); + }, e -> { + log.error("Failed to get threat intel source config [{}]", saTifSourceConfigId); + listener.onFailure(e); + } + )); + } + + private void downloadAndSaveIocsToRefresh(ActionListener listener, SATIFSourceConfig updatedSourceConfig, List stix2IOCList) { + downloadAndSaveIOCs(updatedSourceConfig, stix2IOCList, ActionListener.wrap( + response -> { + // delete old IOCs and update the source config + deleteOldIocIndices(updatedSourceConfig, ActionListener.wrap( + newIocStoreConfig -> { + // Update source config as succeeded, change state back to available + markSourceConfigAsAction(updatedSourceConfig, TIFJobState.AVAILABLE, ActionListener.wrap( + r -> { + log.debug("Set threat intel source config as AVAILABLE for [{}]", updatedSourceConfig.getId()); + SATIFSourceConfigDto returnedSaTifSourceConfigDto = new SATIFSourceConfigDto(updatedSourceConfig); + listener.onResponse(returnedSaTifSourceConfigDto); + }, ex -> { + log.error("Failed to set threat intel source config as AVAILABLE for [{}]", updatedSourceConfig.getId()); + listener.onFailure(ex); + } + )); + }, deleteIocIndicesError -> { + log.error("Failed to delete old IOC indices", deleteIocIndicesError); + listener.onFailure(deleteIocIndicesError); + } + )); + }, downloadAndSaveIocsError -> { + // Update source config as refresh failed + log.error("Failed to download and save IOCs for threat intel source config [{}]", updatedSourceConfig.getId(), downloadAndSaveIocsError); + markSourceConfigAsAction(updatedSourceConfig, TIFJobState.REFRESH_FAILED, ActionListener.wrap( + r -> { + log.info("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSourceConfig.getId()); + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchException( + String.format(Locale.getDefault(), "Failed to download and save IOCs for threat intel source config [%s]. Set source config as REFRESH_FAILED", updatedSourceConfig.getId()), + downloadAndSaveIocsError))); + }, ex -> { + log.error("Failed to set threat intel source config as REFRESH_FAILED for [{}]", updatedSourceConfig.getId()); + listener.onFailure(ex); + } + )); + })); + } + + /** + * @param saTifSourceConfigId + * @param listener + */ + public void deleteTIFSourceConfig( + final String saTifSourceConfigId, + final ActionListener listener + ) { + saTifSourceConfigService.getTIFSourceConfig(saTifSourceConfigId, ActionListener.wrap( + saTifSourceConfig -> { + if (URL_DOWNLOAD.equals(saTifSourceConfig.getType())) { + log.error("Cannot delete tif source config {} as it's a built-in config and not user-defined.", saTifSourceConfigId); + listener.onFailure(new IllegalArgumentException("Cannot delete built-in tif source config " + saTifSourceConfigId)); + return; + } + // Check if all threat intel monitors are deleted + saTifSourceConfigService.checkAndEnsureThreatIntelMonitorsDeleted(ActionListener.wrap( + isDeleted -> { + deleteAllIocsAndSourceConfig(listener, saTifSourceConfig, isDeleted); + }, e -> { + log.error("Failed to check if all threat intel monitors are deleted or if multiple threat intel source configs exist"); + listener.onFailure(e); + } + )); + }, e -> { + log.error("Failed to get threat intel source config for [{}]", saTifSourceConfigId); + if (e instanceof IndexNotFoundException) { + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(), "Threat intel source config [%s] not found.", saTifSourceConfigId), RestStatus.NOT_FOUND))); + } else { + listener.onFailure(e); + } + } + )); + } + + /** + * Deletes the old ioc indices based on retention age and number of indices per index pattern + * + * @param saTifSourceConfig + * @param listener + */ + public void deleteOldIocIndices( + final SATIFSourceConfig saTifSourceConfig, + ActionListener listener + ) { + Set activeIndices = new HashSet<>(); + IocStoreConfig iocStoreConfig = saTifSourceConfig.getIocStoreConfig(); + Set iocIndexPatterns = new HashSet<>(); + if (iocStoreConfig instanceof DefaultIocStoreConfig) { + // get the active indices + DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig(); + defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> activeIndices.add(e.getActiveIndex())); + // get all the index patterns + defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> iocIndexPatterns.add(e.getIndexPattern())); + } + + saTifSourceConfigService.getClusterState(ActionListener.wrap( + clusterStateResponse -> { + Set concreteIndices = SATIFSourceConfigService.getConcreteIndices(clusterStateResponse); + List indicesToDeleteByAge = getIocIndicesToDeleteByAge(clusterStateResponse.getState(), activeIndices); + List indicesToDeleteBySize = getIocIndicesToDeleteBySize( + clusterStateResponse.getState(), + indicesToDeleteByAge.size(), + activeIndices, + concreteIndices); + + Set iocIndicesToDelete = new HashSet<>(); + iocIndicesToDelete.addAll(indicesToDeleteByAge); + iocIndicesToDelete.addAll(indicesToDeleteBySize); + + // delete the indices + saTifSourceConfigService.deleteAllIocIndices(iocIndicesToDelete, true, null); + + // return store config + listener.onResponse(iocStoreConfig); + }, e -> { + log.error("Failed to get the cluster metadata"); + listener.onFailure(e); + } + ), iocIndexPatterns.toArray(new String[0])); + } + + /** + * Helper function to retrieve a list of IOC indices to delete based on retention age + * + * @param clusterState + * @param activeIndices + * @return indicesToDelete + */ + private List getIocIndicesToDeleteByAge( + ClusterState clusterState, + Set activeIndices + ) { + List indicesToDelete = new ArrayList<>(); + Long maxRetentionPeriod = clusterService.getClusterSettings().get(SecurityAnalyticsSettings.IOC_INDEX_RETENTION_PERIOD).millis(); + + for (IndexMetadata indexMetadata : clusterState.metadata().indices().values()) { + Long creationTime = indexMetadata.getCreationDate(); + if ((Instant.now().toEpochMilli() - creationTime) > maxRetentionPeriod) { + String indexToDelete = indexMetadata.getIndex().getName(); + // ensure index is not the current active index + if (activeIndices.contains(indexToDelete) == false) { + indicesToDelete.add(indexToDelete); + } + } + } + return indicesToDelete; + } + + + /** + * Helper function to retrieve a list of IOC indices to delete based on number of indices associated with the index pattern + * + * @param clusterState + * @param totalNumIndicesDeleteByAge + * @param activeIndices + * @param concreteIndices + * @return + */ + private List getIocIndicesToDeleteBySize( + ClusterState clusterState, + Integer totalNumIndicesDeleteByAge, + Set activeIndices, + Set concreteIndices + ) { + Integer numIndicesToDelete = numOfIndicesToDelete(concreteIndices.size(), totalNumIndicesDeleteByAge); + List indicesToDelete = new ArrayList<>(); + + if (numIndicesToDelete > 0) { + // store indices and creation date in map + Map indexToAgeMap = new LinkedHashMap<>(); + final SortedMap lookup = clusterState.getMetadata().getIndicesLookup(); + for (String indexName : concreteIndices) { + IndexAbstraction index = lookup.get(indexName); + IndexMetadata indexMetadata = clusterState.getMetadata().index(indexName); + if (index != null && index.getType() == IndexAbstraction.Type.CONCRETE_INDEX) { + indexToAgeMap.putIfAbsent(indexName, indexMetadata.getCreationDate()); + } + } + + // sort the indexToAgeMap by creation date + List> sortedList = new ArrayList<>(indexToAgeMap.entrySet()); + sortedList.sort(Map.Entry.comparingByValue()); + + // ensure range is not out of bounds + int endIndex = totalNumIndicesDeleteByAge + numIndicesToDelete; + endIndex = Math.min(endIndex, concreteIndices.size()); + + // grab names of indices from totalNumIndicesDeleteByAge to totalNumIndicesDeleteByAge + numIndicesToDelete + for (int i = totalNumIndicesDeleteByAge; i < endIndex; i++) { + // ensure index is not a current active index + if (false == activeIndices.contains(sortedList.get(i).getKey())) { + indicesToDelete.add(sortedList.get(i).getKey()); + } + } + } + return indicesToDelete; + } + + /** + * Helper function to determine how many indices should be deleted based on setting for number of indices per index pattern + * + * @param totalNumIndices + * @param totalNumIndicesDeleteByAge + * @return + */ + private Integer numOfIndicesToDelete(Integer totalNumIndices, Integer totalNumIndicesDeleteByAge) { + Integer maxIndicesPerIndexPattern = clusterService.getClusterSettings().get(SecurityAnalyticsSettings.IOC_MAX_INDICES_PER_INDEX_PATTERN); + Integer numIndicesAfterDeletingByAge = totalNumIndices - totalNumIndicesDeleteByAge; + if (numIndicesAfterDeletingByAge > maxIndicesPerIndexPattern) { + return numIndicesAfterDeletingByAge - maxIndicesPerIndexPattern; + } + return 0; + } + + private void deleteAllIocsAndSourceConfig(ActionListener listener, SATIFSourceConfig saTifSourceConfig, Boolean isDeleted) { + if (isDeleted == false) { + listener.onFailure(new IllegalArgumentException("All threat intel monitors need to be deleted before deleting last threat intel source config")); + } else { + log.debug("All threat intel monitors are deleted or multiple threat intel source configs exist, can delete threat intel source config [{}]", saTifSourceConfig.getId()); + markSourceConfigAsAction( + saTifSourceConfig, + TIFJobState.DELETING, + ActionListener.wrap( + updateSaTifSourceConfigResponse -> { + Set iocIndexPatterns = new HashSet<>(); + if (saTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) { + // get all the index patterns + DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig(); + defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> iocIndexPatterns.add(e.getIndexPattern())); + } + saTifSourceConfigService.getClusterState(ActionListener.wrap( + clusterStateResponse -> { + Set concreteIndices; + if (false == iocIndexPatterns.isEmpty()) { + concreteIndices = SATIFSourceConfigService.getConcreteIndices(clusterStateResponse); + } else { + concreteIndices = new HashSet<>(); + } + saTifSourceConfigService.deleteAllIocIndices(concreteIndices, false, ActionListener.wrap( + r -> { + log.debug("Successfully deleted all ioc indices"); + saTifSourceConfigService.deleteJobSchedulerLockIfJobDisabled(updateSaTifSourceConfigResponse, ActionListener.wrap( + deleteLockResponse -> { + saTifSourceConfigService.deleteTIFSourceConfig(updateSaTifSourceConfigResponse, ActionListener.wrap( + deleteResponse -> { + log.debug("Successfully deleted threat intel source config [{}]", updateSaTifSourceConfigResponse.getId()); + listener.onResponse(deleteResponse); + }, e -> { + log.error("Failed to delete threat intel source config [{}]", saTifSourceConfig.getId()); + listener.onFailure(e); + } + )); + }, e -> { + log.error("Failed to delete threat intel job scheduler lock [{}]", saTifSourceConfig.getId()); + listener.onFailure(e); + } + )); + }, e -> { + log.error("Failed to delete IOC indices for threat intel source config [{}]", updateSaTifSourceConfigResponse.getId()); + listener.onFailure(e); + } + )); + }, e -> { + log.error("Failed to get the cluster metadata"); + listener.onFailure(e); + } + ), iocIndexPatterns.toArray(new String[0])); + }, e -> { + log.error("Failed to update threat intel source config with state as {}", TIFJobState.DELETING); + listener.onFailure(e); + } + )); + } + } + + public void markSourceConfigAsAction(final SATIFSourceConfig saTifSourceConfig, TIFJobState state, ActionListener actionListener) { + TIFJobState previousState = saTifSourceConfig.getState(); + saTifSourceConfig.setState(state); + try { + internalUpdateTIFSourceConfig(saTifSourceConfig, actionListener); + } catch (Exception e) { + log.error("Failed to mark threat intel source config from {} to {} for [{}]", previousState, state, saTifSourceConfig.getId(), e); + actionListener.onFailure(e); + } + } + + /** + * Converts the DTO to entity when creating the source config + * + * @param saTifSourceConfigDto + * @return saTifSourceConfig + */ + public SATIFSourceConfig convertToSATIFConfig(SATIFSourceConfigDto saTifSourceConfigDto, + IocStoreConfig iocStoreConfig, + TIFJobState state, + User createdByUser) { + + // remove duplicates from iocTypes + Set iocTypes = new LinkedHashSet<>(saTifSourceConfigDto.getIocTypes()); + + return new SATIFSourceConfig( + saTifSourceConfigDto.getId(), + saTifSourceConfigDto.getVersion(), + saTifSourceConfigDto.getName(), + saTifSourceConfigDto.getFormat(), + saTifSourceConfigDto.getType(), + saTifSourceConfigDto.getDescription(), + createdByUser, + saTifSourceConfigDto.getCreatedAt(), + saTifSourceConfigDto.getSource(), + saTifSourceConfigDto.getEnabledTime(), + saTifSourceConfigDto.getLastUpdateTime(), + saTifSourceConfigDto.getSchedule(), + state, + saTifSourceConfigDto.getRefreshType(), + saTifSourceConfigDto.getLastRefreshedTime(), + saTifSourceConfigDto.getLastRefreshedUser(), + saTifSourceConfigDto.isEnabled(), + iocStoreConfig, + new ArrayList<>(iocTypes), + saTifSourceConfigDto.isEnabledForScan() + ); + } + + private SATIFSourceConfig updateSaTifSourceConfig(SATIFSourceConfigDto saTifSourceConfigDto, SATIFSourceConfig saTifSourceConfig) { + // currently url download is only for default tif configs and supports only activate/deactivate. Ideally should be via an activate API + if (URL_DOWNLOAD.equals(saTifSourceConfig.getType())) { + return new SATIFSourceConfig( + saTifSourceConfig.getId(), + saTifSourceConfig.getVersion(), + saTifSourceConfig.getName(), + saTifSourceConfig.getFormat(), + saTifSourceConfig.getType(), + saTifSourceConfig.getDescription(), + saTifSourceConfig.getCreatedByUser(), + saTifSourceConfig.getCreatedAt(), + saTifSourceConfig.getSource(), + saTifSourceConfig.getEnabledTime(), + saTifSourceConfig.getLastUpdateTime(), + saTifSourceConfig.getSchedule(), + saTifSourceConfig.getState(), + saTifSourceConfig.getRefreshType(), + saTifSourceConfig.getLastRefreshedTime(), + saTifSourceConfig.getLastRefreshedUser(), + saTifSourceConfig.isEnabled(), + saTifSourceConfig.getIocStoreConfig(), + saTifSourceConfig.getIocTypes(), + saTifSourceConfigDto.isEnabledForScan() + ); + } + if (false == saTifSourceConfig.getSource().getClass().equals(saTifSourceConfigDto.getSource().getClass())) { + throw new IllegalArgumentException(""); + } + // remove duplicates from iocTypes + Set iocTypes = new LinkedHashSet<>(saTifSourceConfigDto.getIocTypes()); + return new SATIFSourceConfig( + saTifSourceConfig.getId(), + saTifSourceConfig.getVersion(), + saTifSourceConfigDto.getName(), + saTifSourceConfigDto.getFormat(), + saTifSourceConfig.getType(), + saTifSourceConfigDto.getDescription(), + saTifSourceConfig.getCreatedByUser(), + saTifSourceConfig.getCreatedAt(), + saTifSourceConfigDto.getSource(), + saTifSourceConfig.getEnabledTime(), + saTifSourceConfig.getLastUpdateTime(), + saTifSourceConfigDto.getSchedule(), + saTifSourceConfig.getState(), + saTifSourceConfigDto.getRefreshType(), + saTifSourceConfig.getLastRefreshedTime(), + saTifSourceConfig.getLastRefreshedUser(), + saTifSourceConfigDto.isEnabled(), + saTifSourceConfig.getIocStoreConfig(), + new ArrayList<>(iocTypes), + saTifSourceConfigDto.isEnabledForScan() + ); + } + + public List convertToIocs(List stix2IocDtoList, String name, String id) { + if (stix2IocDtoList == null) { + return null; + } + return stix2IocDtoList.stream() + .map(dto -> new STIX2IOC(dto, id, name)) + .collect(Collectors.toList()); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java new file mode 100644 index 000000000..dee9ae013 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java @@ -0,0 +1,670 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.service; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.ResourceAlreadyExistsException; +import org.opensearch.action.StepListener; +import org.opensearch.action.admin.cluster.state.ClusterStateRequest; +import org.opensearch.action.admin.cluster.state.ClusterStateResponse; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; +import org.opensearch.action.delete.DeleteRequest; +import org.opensearch.action.delete.DeleteResponse; +import org.opensearch.action.get.GetRequest; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.IndicesOptions; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.routing.Preference; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.jobscheduler.spi.LockModel; +import org.opensearch.search.SearchHit; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.fetch.subphase.FetchSourceContext; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.threatIntel.action.monitor.SearchThreatIntelMonitorAction; +import org.opensearch.securityanalytics.threatIntel.action.monitor.request.SearchThreatIntelMonitorRequest; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.util.IndexUtils; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import org.opensearch.threadpool.ThreadPool; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.opensearch.jobscheduler.spi.utils.LockService.LOCK_INDEX_NAME; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.INDEX_TIMEOUT; +import static org.opensearch.securityanalytics.threatIntel.common.TIFJobState.AVAILABLE; +import static org.opensearch.securityanalytics.threatIntel.common.TIFJobState.REFRESHING; +import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.ENABLED_FOR_SCAN_FIELD; +import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.SOURCE_CONFIG_FIELD; +import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.STATE_FIELD; +import static org.opensearch.securityanalytics.transport.TransportIndexDetectorAction.PLUGIN_OWNER_FIELD; +import static org.opensearch.securityanalytics.util.IndexUtils.shouldUpdateIndex; + +/** + * CRUD for threat intel feeds source config object + */ +public class SATIFSourceConfigService { + private static final Logger log = LogManager.getLogger(SATIFSourceConfigService.class); + private final Client client; + private final ClusterService clusterService; + private final ClusterSettings clusterSettings; + private final ThreadPool threadPool; + private final NamedXContentRegistry xContentRegistry; + private final TIFLockService lockService; + + public SATIFSourceConfigService(final Client client, + final ClusterService clusterService, + ThreadPool threadPool, + NamedXContentRegistry xContentRegistry, + final TIFLockService lockService + ) { + this.client = client; + this.clusterService = clusterService; + this.clusterSettings = clusterService.getClusterSettings(); + this.threadPool = threadPool; + this.xContentRegistry = xContentRegistry; + this.lockService = lockService; + } + + public void indexTIFSourceConfig(SATIFSourceConfig saTifSourceConfig, + final LockModel lock, + final ActionListener actionListener + ) { + StepListener createIndexStepListener = new StepListener<>(); + createIndexStepListener.whenComplete(v -> { + try { + IndexRequest indexRequest = new IndexRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(saTifSourceConfig.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .id(saTifSourceConfig.getId()) + .timeout(clusterSettings.get(INDEX_TIMEOUT)); + + log.debug("Indexing tif source config"); + client.index(indexRequest, ActionListener.wrap( + response -> { + log.debug("Threat intel source config with id [{}] indexed success.", response.getId()); + SATIFSourceConfig responseSaTifSourceConfig = createSATIFSourceConfig(saTifSourceConfig, response); + actionListener.onResponse(responseSaTifSourceConfig); + }, e -> { + log.error("Failed to index threat intel source config with id [{}]", saTifSourceConfig.getId()); + actionListener.onFailure(e); + }) + ); + + } catch (IOException e) { + log.error("Exception saving the threat intel source config in index", e); + actionListener.onFailure(e); + } + }, exception -> { + log.error("Failed to create threat intel source config index", exception); + lockService.releaseLock(lock, ActionListener.wrap( + r -> { + log.debug("Released threat intel source config lock with id [{}]", lock.getLockId()); + actionListener.onFailure(exception); + }, + ex -> { + log.error(String.format("Unexpected failure while trying to release lock [%s] for threat intel source config.", lock.getLockId()), ex); + actionListener.onFailure(exception); + } + )); + }); + createJobIndexIfNotExists(createIndexStepListener); + } + + private static SATIFSourceConfig createSATIFSourceConfig(SATIFSourceConfig saTifSourceConfig, IndexResponse response) { + return new SATIFSourceConfig( + response.getId(), + response.getVersion(), + saTifSourceConfig.getName(), + saTifSourceConfig.getFormat(), + saTifSourceConfig.getType(), + saTifSourceConfig.getDescription(), + saTifSourceConfig.getCreatedByUser(), + saTifSourceConfig.getCreatedAt(), + saTifSourceConfig.getSource(), + saTifSourceConfig.getEnabledTime(), + saTifSourceConfig.getLastUpdateTime(), + saTifSourceConfig.getSchedule(), + saTifSourceConfig.getState(), + saTifSourceConfig.getRefreshType(), + saTifSourceConfig.getLastRefreshedTime(), + saTifSourceConfig.getLastRefreshedUser(), + saTifSourceConfig.isEnabled(), + saTifSourceConfig.getIocStoreConfig(), + saTifSourceConfig.getIocTypes(), + saTifSourceConfig.isEnabledForScan() + ); + } + + // Get the job config index mapping + private String getIndexMapping() { + try { + try (InputStream is = SATIFSourceConfigService.class.getResourceAsStream("/mappings/threat_intel_job_mapping.json")) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + return reader.lines().map(String::trim).collect(Collectors.joining()); + } + } + } catch (IOException e) { + log.error("Failed to get the threat intel index mapping", e); + throw new SecurityAnalyticsException("Failed to get threat intel index mapping", RestStatus.BAD_REQUEST, e); + } + } + + // Create TIF source config index + + /** + * Index name: .opensearch-sap--job + * Mapping: /mappings/threat_intel_job_mapping.json + * Updates the job index mapping if currently on a previous version + * + * @param stepListener setup listener + */ + public void createJobIndexIfNotExists(final StepListener stepListener) { + // check if job index exists + if (clusterService.state().metadata().hasIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME) == true) { + checkAndUpdateJobIndexMapping(stepListener); + } else { + final CreateIndexRequest createIndexRequest = new CreateIndexRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME).mapping(getIndexMapping()) + .settings(SecurityAnalyticsPlugin.TIF_JOB_INDEX_SETTING); + client.admin().indices().create(createIndexRequest, ActionListener.wrap( + r -> { + log.debug("[{}] index created", SecurityAnalyticsPlugin.JOB_INDEX_NAME); + stepListener.onResponse(null); + }, e -> { + if (e instanceof ResourceAlreadyExistsException) { + log.info("Index [{}] already exists", SecurityAnalyticsPlugin.JOB_INDEX_NAME); + stepListener.onResponse(null); + return; + } + log.error("Failed to create [{}] index", SecurityAnalyticsPlugin.JOB_INDEX_NAME, e); + stepListener.onFailure(e); + } + )); + } + } + + private void checkAndUpdateJobIndexMapping(StepListener stepListener) { + try { + // Check if job index contains old mapping, if so update index mapping (current version = 2) + if (shouldUpdateIndex(clusterService.state().metadata().index(SecurityAnalyticsPlugin.JOB_INDEX_NAME), getIndexMapping())) { + log.info("Old schema version found for [{}] index, updating the index mapping", SecurityAnalyticsPlugin.JOB_INDEX_NAME); + IndexUtils.updateIndexMapping( + SecurityAnalyticsPlugin.JOB_INDEX_NAME, + getIndexMapping(), clusterService.state(), client.admin().indices(), + ActionListener.wrap( + r -> { + log.info("Successfully updated index mapping for [{}] index", SecurityAnalyticsPlugin.JOB_INDEX_NAME); + stepListener.onResponse(null); + }, e -> { + // Check if version is updated despite failure + try { + // Check if job index still contains older mapping + if (shouldUpdateIndex(clusterService.state().metadata().index(SecurityAnalyticsPlugin.JOB_INDEX_NAME), getIndexMapping())) { + log.error("Job index still contains older mapping, failed to update job index mapping", e); + stepListener.onFailure(e); + } else { + // If job index contains newest mapping, then return success + log.info("Successfully updated index mapping for [{}] index", SecurityAnalyticsPlugin.JOB_INDEX_NAME); + stepListener.onResponse(null); + } + } catch (IOException exception) { + log.error("Failed to check if job index contains older mapping. Failed to update job index mapping", e); + stepListener.onFailure(e); + } + } + ), + false + ); + } else { + // If job index contains newest mapping, then do nothing + stepListener.onResponse(null); + } + } catch (IOException e) { + log.error("Failed to check and update job index mapping", e); + stepListener.onFailure(e); + } + } + + // Get TIF source config + public void getTIFSourceConfig( + String tifSourceConfigId, + ActionListener actionListener + ) { + GetRequest getRequest = new GetRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME, tifSourceConfigId); + client.get(getRequest, ActionListener.wrap( + getResponse -> { + if (!getResponse.isExists()) { + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.ROOT, "Threat intel source config [%s] not found.", tifSourceConfigId), RestStatus.NOT_FOUND))); + return; + } + SATIFSourceConfig saTifSourceConfig = null; + if (!getResponse.isSourceEmpty()) { + XContentParser xcp = XContentHelper.createParser( + xContentRegistry, LoggingDeprecationHandler.INSTANCE, + getResponse.getSourceAsBytesRef(), XContentType.JSON + ); + saTifSourceConfig = SATIFSourceConfig.docParse(xcp, getResponse.getId(), getResponse.getVersion()); + } + if (saTifSourceConfig == null) { + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.ROOT, "No threat intel source config exists [%s]", tifSourceConfigId), RestStatus.BAD_REQUEST))); + } else { + log.debug("Threat intel source config with id [{}] fetched", getResponse.getId()); + actionListener.onResponse(saTifSourceConfig); + } + }, e -> { + log.error("Failed to fetch threat intel source config document", e); + actionListener.onFailure(e); + }) + ); + } + + public void searchTIFSourceConfigs( + final SearchSourceBuilder searchSourceBuilder, + final ActionListener actionListener + ) { + SearchRequest searchRequest = getSearchRequest(searchSourceBuilder); + + client.search(searchRequest, ActionListener.wrap( + searchResponse -> { + if (searchResponse.isTimedOut()) { + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("Search threat intel source configs request timed out", RestStatus.REQUEST_TIMEOUT))); + return; + } + + // convert search hits to threat intel source configs + for (SearchHit hit : searchResponse.getHits()) { + XContentParser xcp = XContentType.JSON.xContent().createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString() + ); + SATIFSourceConfig satifSourceConfig = SATIFSourceConfig.docParse(xcp, hit.getId(), hit.getVersion()); + XContentBuilder xcb = satifSourceConfig.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS); + hit.sourceRef(BytesReference.bytes(xcb)); + } + + log.debug("Fetched all threat intel source configs successfully."); + actionListener.onResponse(searchResponse); + }, e -> { + log.error("Failed to fetch all threat intel source configs for search request [{}]", searchRequest, e); + actionListener.onFailure(e); + }) + ); + } + + private static SearchRequest getSearchRequest(SearchSourceBuilder searchSourceBuilder) { + + // update search source builder + searchSourceBuilder.seqNoAndPrimaryTerm(true); + searchSourceBuilder.version(true); + + // construct search request + SearchRequest searchRequest = new SearchRequest().source(searchSourceBuilder); + searchRequest.indices(SecurityAnalyticsPlugin.JOB_INDEX_NAME); + searchRequest.preference(Preference.PRIMARY_FIRST.type()); + + BoolQueryBuilder boolQueryBuilder; + + if (searchRequest.source().query() == null) { + boolQueryBuilder = new BoolQueryBuilder(); + } else { + boolQueryBuilder = QueryBuilders.boolQuery().must(searchRequest.source().query()); + } + + BoolQueryBuilder bqb = new BoolQueryBuilder(); + bqb.should().add(new BoolQueryBuilder().must(QueryBuilders.existsQuery("source_config"))); + + boolQueryBuilder.filter(bqb); + searchRequest.source().query(boolQueryBuilder); + return searchRequest; + } + + // Update TIF source config + public void updateTIFSourceConfig( + SATIFSourceConfig saTifSourceConfig, + final ActionListener actionListener + ) { + try { + IndexRequest indexRequest = new IndexRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(saTifSourceConfig.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .id(saTifSourceConfig.getId()) + .timeout(clusterSettings.get(INDEX_TIMEOUT)); + + client.index(indexRequest, ActionListener.wrap(response -> { + log.debug("Threat intel source config with id [{}] update success.", response.getId()); + SATIFSourceConfig responseSaTifSourceConfig = createSATIFSourceConfig(saTifSourceConfig, response); + actionListener.onResponse(responseSaTifSourceConfig); + }, e -> { + log.error("Failed to index threat intel source config with id [{}]", saTifSourceConfig.getId()); + actionListener.onFailure(e); + }) + ); + + } catch (IOException e) { + log.error("Exception updating the threat intel source config in index", e); + } + } + + // Delete TIF source config + public void deleteTIFSourceConfig( + SATIFSourceConfig saTifSourceConfig, + final ActionListener actionListener + ) { + // check to make sure the job index exists + if (clusterService.state().metadata().hasIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME) == false) { + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("Threat intel source config index does not exist", RestStatus.BAD_REQUEST))); + return; + } + + DeleteRequest request = new DeleteRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME, saTifSourceConfig.getId()) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .timeout(clusterSettings.get(INDEX_TIMEOUT)); + + client.delete(request, ActionListener.wrap( + deleteResponse -> { + if (deleteResponse.status().equals(RestStatus.OK)) { + log.info("Deleted threat intel source config [{}] successfully", saTifSourceConfig.getId()); + actionListener.onResponse(deleteResponse); + } else if (deleteResponse.status().equals(RestStatus.NOT_FOUND)) { + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.ROOT, "Threat intel source config with id [{%s}] not found", saTifSourceConfig.getId()), RestStatus.NOT_FOUND))); + } else { + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.ROOT, "Failed to delete threat intel source config [{%s}]", saTifSourceConfig.getId()), deleteResponse.status()))); + } + }, e -> { + log.error("Failed to delete threat intel source config with id [{}]", saTifSourceConfig.getId()); + actionListener.onFailure(e); + } + )); + } + + // Manually delete threat intel job scheduler lock if job is disabled + public void deleteJobSchedulerLockIfJobDisabled( + SATIFSourceConfig saTifSourceConfig, + final ActionListener actionListener + ) { + if (saTifSourceConfig.isEnabled()) { + actionListener.onResponse(null); + return; + } + + // check to make sure the job scheduler lock index exists + if (clusterService.state().metadata().hasIndex(LOCK_INDEX_NAME) == false) { + actionListener.onResponse(null); + return; + } + + String id = SecurityAnalyticsPlugin.JOB_INDEX_NAME + "-" + saTifSourceConfig.getId(); + DeleteRequest request = new DeleteRequest(LOCK_INDEX_NAME, id) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .timeout(clusterSettings.get(INDEX_TIMEOUT)); + + client.delete(request, ActionListener.wrap( + deleteResponse -> { + if (deleteResponse.status().equals(RestStatus.OK)) { + log.info("Deleted threat intel job scheduler lock [{}] successfully", id); + actionListener.onResponse(deleteResponse); + } else if (deleteResponse.status().equals(RestStatus.NOT_FOUND)) { + log.info("Threat intel job scheduler lock with id [{}] not found", id); + actionListener.onResponse(deleteResponse); + } else { + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.ROOT, "Failed to delete threat intel job scheduler lock with id [{%s}]", id), deleteResponse.status()))); + } + }, e -> { + log.error("Failed to delete threat intel job scheduler lock with id [{}]", id); + actionListener.onFailure(e); + } + )); + } + + public void deleteAllIocIndices(Set indicesToDelete, Boolean backgroundJob, ActionListener listener) { + if (indicesToDelete.isEmpty() == false) { + DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indicesToDelete.toArray(new String[0])); + client.admin().indices().delete( + deleteIndexRequest, + ActionListener.wrap( + deleteIndicesResponse -> { + if (!deleteIndicesResponse.isAcknowledged()) { + log.error("Could not delete one or more IOC indices: [" + indicesToDelete + "]. Retrying one by one."); + deleteIocIndex(indicesToDelete, backgroundJob, listener); + } else { + log.info("Successfully deleted indices: [" + indicesToDelete + "]"); + if (backgroundJob == false) { + listener.onResponse(deleteIndicesResponse); + } + } + }, e -> { + log.error("Delete for IOC Indices failed: [" + indicesToDelete + "]. Retrying one By one."); + deleteIocIndex(indicesToDelete, backgroundJob, listener); + } + ) + ); + } else if (listener != null) { + listener.onResponse(new AcknowledgedResponse(true)); + } + } + + private void deleteIocIndex(Set indicesToDelete, Boolean backgroundJob, ActionListener listener) { + for (String index : indicesToDelete) { + final DeleteIndexRequest singleDeleteRequest = new DeleteIndexRequest(indicesToDelete.toArray(new String[0])); + client.admin().indices().delete( + singleDeleteRequest, + ActionListener.wrap( + response -> { + if (!response.isAcknowledged()) { + log.error("Could not delete one or more IOC indices: " + index); + if (backgroundJob == false) { + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.ROOT, "Could not delete one or more IOC indices: " + index), RestStatus.BAD_REQUEST))); + } + } else { + log.debug("Successfully deleted one or more IOC indices:" + index); + if (backgroundJob == false) { + listener.onResponse(response); + } + } + }, e -> { + log.debug("Exception: [" + e.getMessage() + "] while deleting the index " + index); + if (backgroundJob == false) { + listener.onFailure(e); + } + } + ) + ); + } + } + + public void getClusterState( + final ActionListener actionListener, + String... indices) { + ClusterStateRequest clusterStateRequest = new ClusterStateRequest() + .clear() + .indices(indices) + .metadata(true) + .local(true) + .indicesOptions(IndicesOptions.strictExpand()); + client.admin().cluster().state( + clusterStateRequest, + ActionListener.wrap( + clusterStateResponse -> { + log.debug("Successfully retrieved cluster state"); + actionListener.onResponse(clusterStateResponse); + }, e -> { + log.error("Error fetching cluster state"); + actionListener.onFailure(e); + } + ) + ); + } + + public void checkAndEnsureThreatIntelMonitorsDeleted( + ActionListener listener + ) { + // TODO: change this to use search source configs API call + SearchRequest searchRequest = new SearchRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME) + .source(new SearchSourceBuilder() + .seqNoAndPrimaryTerm(false) + .version(false) + .query(QueryBuilders.matchAllQuery()) + .fetchSource(FetchSourceContext.FETCH_SOURCE) + ).preference(Preference.PRIMARY_FIRST.type()); + + // Search if there is only one threat intel source config left + client.search(searchRequest, ActionListener.wrap( + saTifSourceConfigResponse -> { + if (saTifSourceConfigResponse.getHits().getHits().length <= 1) { + String alertingConfigIndex = ".opendistro-alerting-config"; + if (clusterService.state().metadata().hasIndex(alertingConfigIndex) == false) { + log.debug("[{}] index does not exist, continuing deleting threat intel source config", alertingConfigIndex); + listener.onResponse(true); + } else { + // Search alerting config index for at least one threat intel monitor + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .seqNoAndPrimaryTerm(false) + .version(false) + .query(QueryBuilders.matchAllQuery()) + .fetchSource(FetchSourceContext.FETCH_SOURCE); + + SearchRequest newSearchRequest = new SearchRequest(); + newSearchRequest.source(searchSourceBuilder); + newSearchRequest.indices(alertingConfigIndex); + newSearchRequest.preference(Preference.PRIMARY_FIRST.type()); + + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().must(newSearchRequest.source().query()); + BoolQueryBuilder bqb = new BoolQueryBuilder(); + bqb.should().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.owner", PLUGIN_OWNER_FIELD))); + boolQueryBuilder.filter(bqb); + newSearchRequest.source().query(boolQueryBuilder); // TODO: remove this once logic is moved to transport layer + + client.execute(SearchThreatIntelMonitorAction.INSTANCE, new SearchThreatIntelMonitorRequest(newSearchRequest), ActionListener.wrap( + response -> { + if (response.getHits().getHits().length == 0) { + log.debug("All threat intel monitors are deleted, continuing deleting threat intel source config"); + listener.onResponse(true); + } else { + log.error("All threat intel monitors need to be deleted before deleting threat intel source config"); + listener.onResponse(false); + } + }, e -> { + log.error("Failed to search for threat intel monitors"); + listener.onFailure(e); + } + )); + } + } else { + // If there are multiple threat intel source configs left, proceed with deletion + log.debug("Multiple threat intel source configs exist, threat intel monitors do not need to be deleted"); + listener.onResponse(true); + } + }, e -> { + log.error("Failed to search for threat intel source configs"); + listener.onFailure(e); + } + )); + + } + + /** + * Returns a map of ioc type to a list of active indices + * + * @param listener + */ + public void getIocTypeToIndices(ActionListener>> listener) { + SearchRequest searchRequest = new SearchRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME); + + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + queryBuilder.must(QueryBuilders.termQuery(getEnabledForScanFieldName(), true)); + + String stateFieldName = getStateFieldName(); + BoolQueryBuilder stateQueryBuilder = QueryBuilders.boolQuery() + .should(QueryBuilders.matchQuery(stateFieldName, AVAILABLE.toString())); + stateQueryBuilder.should(QueryBuilders.matchQuery(stateFieldName, REFRESHING.toString())); + queryBuilder.must(stateQueryBuilder); + + searchRequest.source().query(queryBuilder); + client.search(searchRequest, ActionListener.wrap( + searchResponse -> { + Map> cumulativeIocTypeToIndices = new HashMap<>(); + for (SearchHit hit : searchResponse.getHits().getHits()) { + XContentParser xcp = XContentType.JSON.xContent().createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString() + ); + SATIFSourceConfig config = SATIFSourceConfig.docParse(xcp, hit.getId(), hit.getVersion()); + if (config.getIocStoreConfig() instanceof DefaultIocStoreConfig) { + DefaultIocStoreConfig iocStoreConfig = (DefaultIocStoreConfig) config.getIocStoreConfig(); + for (DefaultIocStoreConfig.IocToIndexDetails iocToindexDetails : iocStoreConfig.getIocToIndexDetails()) { + String activeIndex = iocToindexDetails.getActiveIndex(); + IOCType iocType = iocToindexDetails.getIocType(); + List strings = cumulativeIocTypeToIndices.computeIfAbsent(iocType.toString(), k -> new ArrayList<>()); + strings.add(activeIndex); + } + } + } + listener.onResponse(cumulativeIocTypeToIndices); + }, + e -> { + log.error("Failed to fetch ioc indices", e); + listener.onFailure(e); + } + )); + } + + public static String getStateFieldName() { + return String.format("%s.%s", SOURCE_CONFIG_FIELD, STATE_FIELD); + } + + + public static String getEnabledForScanFieldName() { + return String.format("%s.%s", SOURCE_CONFIG_FIELD, ENABLED_FOR_SCAN_FIELD); + } + + public static Set getConcreteIndices(ClusterStateResponse clusterStateResponse) { + Set concreteIndices = new HashSet<>(); + Collection values = clusterStateResponse.getState().metadata().indices().values(); + for (IndexMetadata metadata : values) { + concreteIndices.add(metadata.getIndex().getName()); + } + return concreteIndices; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobParameterService.java similarity index 96% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterService.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobParameterService.java index 55387cb35..c7fa5566e 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobParameterService.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel.jobscheduler; +package org.opensearch.securityanalytics.threatIntel.service; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -15,10 +15,8 @@ import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexResponse; import org.opensearch.action.get.GetRequest; -import org.opensearch.action.get.GetResponse; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.WriteRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; @@ -30,11 +28,10 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.IndexNotFoundException; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; -import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.action.ThreatIntelIndicesResponse; import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import java.io.BufferedReader; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobUpdateService.java similarity index 96% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateService.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobUpdateService.java index 5c48ed8aa..ab6ed9915 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobUpdateService.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel.jobscheduler; +package org.opensearch.securityanalytics.threatIntel.service; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; @@ -15,16 +15,15 @@ import org.opensearch.OpenSearchStatusException; import org.opensearch.action.admin.indices.create.CreateIndexResponse; import org.opensearch.action.support.GroupedActionListener; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.core.action.ActionListener; import org.opensearch.core.rest.RestStatus; -import org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataService; -import org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedParser; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.util.ThreatIntelFeedParser; import org.opensearch.securityanalytics.threatIntel.action.ThreatIntelIndicesResponse; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; @@ -188,7 +187,7 @@ public void onFailure(Exception e) { } break; default: - // if the feed type doesn't match any of the supporting feed types, throw an exception + onFailure(new UnsupportedOperationException("Not a supported feed format : " + tifMetadata.getFeedType())); } } } catch (IOException ex) { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelFeedDataService.java similarity index 97% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelFeedDataService.java index b9d8aa3ea..1cb9e7428 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelFeedDataService.java @@ -2,7 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel; +package org.opensearch.securityanalytics.threatIntel.service; import org.apache.commons.csv.CSVRecord; import org.apache.commons.lang3.StringUtils; @@ -20,7 +20,6 @@ import org.opensearch.action.support.GroupedActionListener; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.WriteRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; @@ -39,8 +38,8 @@ import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobRequest; import org.opensearch.securityanalytics.threatIntel.action.ThreatIntelIndicesResponse; import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.util.ThreatIntelFeedDataUtils; import org.opensearch.securityanalytics.util.IndexUtils; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; @@ -51,7 +50,6 @@ import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.ArrayList; -import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -60,7 +58,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; -import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; +import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; /** * Service to handle CRUD operations on Threat Intel Feed Data @@ -231,6 +229,8 @@ public void parseAndSaveThreatIntelFeedDataCSV( } public static boolean isValidIp(String ip) { + if (StringUtils.isBlank(ip)) + return false; String ipPattern = "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}$"; Pattern pattern = Pattern.compile(ipPattern); Matcher matcher = pattern.matcher(ip); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportDeleteTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportDeleteTIFSourceConfigAction.java new file mode 100644 index 000000000..58b1e5bc1 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportDeleteTIFSourceConfigAction.java @@ -0,0 +1,68 @@ +package org.opensearch.securityanalytics.threatIntel.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.SADeleteTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.SADeleteTIFSourceConfigRequest; +import org.opensearch.securityanalytics.threatIntel.action.SADeleteTIFSourceConfigResponse; +import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService; +import org.opensearch.securityanalytics.transport.SecureTransportAction; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +public class TransportDeleteTIFSourceConfigAction extends HandledTransportAction implements SecureTransportAction { + + private static final Logger log = LogManager.getLogger(TransportDeleteTIFSourceConfigAction.class); + + private final Settings settings; + private final ThreadPool threadPool; + private volatile Boolean filterByEnabled; + private final SATIFSourceConfigManagementService saTifConfigService; + + @Inject + public TransportDeleteTIFSourceConfigAction(TransportService transportService, + ActionFilters actionFilters, + Settings settings, + final ThreadPool threadPool, + final SATIFSourceConfigManagementService saTifConfigService) { + super(SADeleteTIFSourceConfigAction.NAME, transportService, actionFilters, SADeleteTIFSourceConfigRequest::new); + this.settings = settings; + this.threadPool = threadPool; + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.saTifConfigService = saTifConfigService; + } + + @Override + protected void doExecute(Task task, SADeleteTIFSourceConfigRequest request, ActionListener actionListener) { + User user = readUserFromThreadContext(this.threadPool); + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(validateBackendRoleMessage, RestStatus.FORBIDDEN))); + return; + } + this.threadPool.getThreadContext().stashContext(); + + saTifConfigService.deleteTIFSourceConfig(request.getId(), ActionListener.wrap( + response -> actionListener.onResponse( + new SADeleteTIFSourceConfigResponse( + request.getId(), + response.status() + ) + ), e -> { + log.error("Failed to delete threat intel source config [{}] ", request.getId()); + actionListener.onFailure(e); + }) + ); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java new file mode 100644 index 000000000..4e6d2f349 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java @@ -0,0 +1,190 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.transport; + +import org.apache.lucene.search.join.ScoreMode; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.Strings; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.Operator; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.SearchHit; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.fetch.subphase.FetchSourceContext; +import org.opensearch.search.sort.FieldSortBuilder; +import org.opensearch.search.sort.SortBuilders; +import org.opensearch.search.sort.SortOrder; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsAction; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsRequest; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsResponse; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService; +import org.opensearch.securityanalytics.transport.SecureTransportAction; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +public class TransportGetIocFindingsAction extends HandledTransportAction implements SecureTransportAction { + + private final IocFindingService iocFindingService; + + private final ClusterService clusterService; + + private final Settings settings; + + private final NamedXContentRegistry xContentRegistry; + private final ThreadPool threadPool; + + private volatile Boolean filterByEnabled; + + @Inject + public TransportGetIocFindingsAction( + TransportService transportService, + ActionFilters actionFilters, + ClusterService clusterService, + Settings settings, + NamedXContentRegistry xContentRegistry, + Client client + ) { + super(GetIocFindingsAction.NAME, transportService, actionFilters, GetIocFindingsRequest::new); + this.settings = settings; + this.clusterService = clusterService; + this.xContentRegistry = xContentRegistry; + this.threadPool = client.threadPool(); + this.iocFindingService = new IocFindingService(client, this.clusterService, xContentRegistry); + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled); + } + + @Override + protected void doExecute(Task task, GetIocFindingsRequest request, ActionListener actionListener) { + User user = readUserFromThreadContext(this.threadPool); + + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + actionListener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); + return; + } + this.threadPool.getThreadContext().stashContext(); + + Table tableProp = request.getTable(); + FieldSortBuilder sortBuilder = SortBuilders + .fieldSort(tableProp.getSortString()) + .order(SortOrder.fromString(tableProp.getSortOrder())); + if (tableProp.getMissing() != null && !tableProp.getMissing().isBlank()) { + sortBuilder.missing(tableProp.getMissing()); + } + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .sort(sortBuilder) + .size(tableProp.getSize()) + .from(tableProp.getStartIndex()) + .fetchSource(new FetchSourceContext(true, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY)) + .seqNoAndPrimaryTerm(true) + .version(true); + + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + List findingIds = request.getFindingIds(); + + if (findingIds != null && !findingIds.isEmpty()) { + BoolQueryBuilder findingIdsFilter = QueryBuilders.boolQuery(); + findingIds.forEach(it -> findingIdsFilter.should(QueryBuilders.matchQuery("_id", it))); + queryBuilder.filter(findingIdsFilter); + } + + List iocIds = request.getIocIds(); + if (iocIds != null && !iocIds.isEmpty()) { + BoolQueryBuilder iocIdQueryBuilder = QueryBuilders.boolQuery(); + // can't use match query because it analyzes the value and considers `hyphens` as word separators + iocIds.forEach(it -> iocIdQueryBuilder.should(QueryBuilders.matchPhraseQuery("ioc_feed_ids.ioc_id", it))); + queryBuilder.filter(iocIdQueryBuilder); + } + + Instant startTime = request.getStartTime(); + Instant endTime = request.getEndTime(); + if (startTime != null && endTime != null) { + long startTimeMillis = startTime.toEpochMilli(); + long endTimeMillis = endTime.toEpochMilli(); + QueryBuilder timeRangeQuery = QueryBuilders.rangeQuery("timestamp") + .from(startTimeMillis) // Greater than or equal to start time + .to(endTimeMillis); // Less than or equal to end time + queryBuilder.filter(timeRangeQuery); + } + + if (tableProp.getSearchString() != null && !tableProp.getSearchString().isBlank()) { + queryBuilder.should(QueryBuilders + .queryStringQuery(tableProp.getSearchString()) + ).should( + QueryBuilders.nestedQuery( + "queries", + QueryBuilders.boolQuery() + .must( + QueryBuilders.queryStringQuery(tableProp.getSearchString()) + ), + ScoreMode.Avg + ) + ); + } + searchSourceBuilder.query(queryBuilder).trackTotalHits(true); + + this.threadPool.getThreadContext().stashContext(); + iocFindingService.search(searchSourceBuilder, new ActionListener<>() { + @Override + public void onResponse(SearchResponse searchResponse) { + try { + long totalIocFindingsCount = searchResponse.getHits().getTotalHits().value; + List iocFindings = new ArrayList<>(); + + for (SearchHit hit : searchResponse.getHits()) { + XContentParser xcp = XContentType.JSON.xContent() + .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString()); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + IocFinding iocFinding = IocFinding.parse(xcp); + iocFindings.add(iocFinding); + } + actionListener.onResponse(new GetIocFindingsResponse((int) totalIocFindingsCount, iocFindings)); + } catch (Exception ex) { + this.onFailure(ex); + } + } + + @Override + public void onFailure(Exception e) { + if (e instanceof IndexNotFoundException) { + actionListener.onResponse(new GetIocFindingsResponse(0, List.of())); + return; + } + actionListener.onFailure(e); + } + }); + } + + private void setFilterByEnabled(boolean filterByEnabled) { + this.filterByEnabled = filterByEnabled; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetTIFSourceConfigAction.java new file mode 100644 index 000000000..51c5a6ad2 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetTIFSourceConfigAction.java @@ -0,0 +1,84 @@ +package org.opensearch.securityanalytics.threatIntel.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigRequest; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigResponse; +import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService; +import org.opensearch.securityanalytics.transport.SecureTransportAction; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +public class TransportGetTIFSourceConfigAction extends HandledTransportAction implements SecureTransportAction { + + private static final Logger log = LogManager.getLogger(TransportGetTIFSourceConfigAction.class); + + private final ClusterService clusterService; + + private final Settings settings; + + private final ThreadPool threadPool; + + private volatile Boolean filterByEnabled; + + private final SATIFSourceConfigManagementService saTifConfigService; + + @Inject + public TransportGetTIFSourceConfigAction(TransportService transportService, + ActionFilters actionFilters, + ClusterService clusterService, + final ThreadPool threadPool, + Settings settings, + final SATIFSourceConfigManagementService saTifConfigService) { + super(SAGetTIFSourceConfigAction.NAME, transportService, actionFilters, SAGetTIFSourceConfigRequest::new); + this.clusterService = clusterService; + this.threadPool = threadPool; + this.settings = settings; + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled); + this.saTifConfigService = saTifConfigService; + } + + @Override + protected void doExecute(Task task, SAGetTIFSourceConfigRequest request, ActionListener actionListener) { + // validate user + User user = readUserFromThreadContext(this.threadPool); + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + actionListener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); + return; + } + this.threadPool.getThreadContext().stashContext(); + + saTifConfigService.getTIFSourceConfig(request.getId(), ActionListener.wrap( + saTifSourceConfigDtoResponse -> actionListener.onResponse( + new SAGetTIFSourceConfigResponse( + saTifSourceConfigDtoResponse.getId(), + saTifSourceConfigDtoResponse.getVersion(), + RestStatus.OK, + saTifSourceConfigDtoResponse + ) + ), e -> { + log.error("Failed to get threat intel source config for [{}]", request.getId()); + actionListener.onFailure(e); + }) + ); + } + + private void setFilterByEnabled(boolean filterByEnabled) { + this.filterByEnabled = filterByEnabled; + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java new file mode 100644 index 000000000..afd22f799 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java @@ -0,0 +1,169 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.rest.RestRequest; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigRequest; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigResponse; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService; +import org.opensearch.securityanalytics.transport.SecureTransportAction; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.util.ConcurrentModificationException; + +import static org.opensearch.securityanalytics.threatIntel.common.TIFLockService.LOCK_DURATION_IN_SECONDS; + +/** + * Transport action to create threat intel feeds source config object and save IoCs + */ +public class TransportIndexTIFSourceConfigAction extends HandledTransportAction implements SecureTransportAction { + private static final Logger log = LogManager.getLogger(TransportIndexTIFSourceConfigAction.class); + private final SATIFSourceConfigManagementService saTifSourceConfigManagementService; + private final TIFLockService lockService; + private final ThreadPool threadPool; + private final Settings settings; + private volatile Boolean filterByEnabled; + + /** + * Default constructor + * + * @param transportService the transport service + * @param actionFilters the action filters + * @param threadPool the thread pool + * @param lockService the lock service + */ + @Inject + public TransportIndexTIFSourceConfigAction( + final TransportService transportService, + final ActionFilters actionFilters, + final ThreadPool threadPool, + final SATIFSourceConfigManagementService saTifSourceConfigManagementService, + final TIFLockService lockService, + final Settings settings + ) { + super(SAIndexTIFSourceConfigAction.NAME, transportService, actionFilters, SAIndexTIFSourceConfigRequest::new); + this.threadPool = threadPool; + this.saTifSourceConfigManagementService = saTifSourceConfigManagementService; + this.lockService = lockService; + this.settings = settings; + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + } + + + @Override + protected void doExecute(final Task task, final SAIndexTIFSourceConfigRequest request, final ActionListener listener) { + // validate user + User user = readUserFromThreadContext(this.threadPool); + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + + if (!"".equals(validateBackendRoleMessage)) { + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(validateBackendRoleMessage, RestStatus.FORBIDDEN))); + return; + } + this.threadPool.getThreadContext().stashContext(); + + retrieveLockAndCreateTIFConfig(request, listener, user); + } + + private void retrieveLockAndCreateTIFConfig(SAIndexTIFSourceConfigRequest request, ActionListener listener, User user) { + try { + lockService.acquireLock(request.getTIFConfigDto().getId(), LOCK_DURATION_IN_SECONDS, ActionListener.wrap(lock -> { + if (lock == null) { + log.error("another processor is a lock, BAD_REQUEST error", RestStatus.BAD_REQUEST); + listener.onFailure( + new ConcurrentModificationException("another processor is holding a lock on the resource. Try again later") + ); + return; + } + try { + SATIFSourceConfigDto saTifSourceConfigDto = request.getTIFConfigDto(); + saTifSourceConfigManagementService.createOrUpdateTifSourceConfig( + saTifSourceConfigDto, + lock, + request.getMethod(), + user, + ActionListener.wrap( + saTifSourceConfigDtoResponse -> { + lockService.releaseLock(lock, ActionListener.wrap( + r -> { + log.debug("Released threat intel source config lock with id [{}]", lock.getLockId()); + listener.onResponse(new SAIndexTIFSourceConfigResponse( + saTifSourceConfigDtoResponse.getId(), + saTifSourceConfigDtoResponse.getVersion(), + RestStatus.OK, + saTifSourceConfigDtoResponse + )); + }, + e -> { + log.error(String.format("Unexpected failure while trying to release lock [%s] for tif source config [%s].", lock.getLockId(), saTifSourceConfigDto.getId()), e); + listener.onResponse(new SAIndexTIFSourceConfigResponse( + saTifSourceConfigDtoResponse.getId(), + saTifSourceConfigDtoResponse.getVersion(), + RestStatus.OK, + saTifSourceConfigDtoResponse + )); + } + )); + }, e -> { + String action = RestRequest.Method.PUT.equals(request.getMethod()) ? "update" : "create"; + log.error(String.format("Failed to %s IOCs and threat intel source config", action), e); + lockService.releaseLock(lock, ActionListener.wrap( + r -> { + log.debug("Released threat intel source config lock with id [{}]", lock.getLockId()); + listener.onFailure(e); + }, + ex -> { + log.error(String.format("Unexpected failure while trying to release lock [%s] for tif source config.", lock.getLockId()), ex); + listener.onFailure(e); + } + )); + } + + ) + ); + } catch (Exception e) { + String action = RestRequest.Method.PUT.equals(request.getMethod()) ? "update" : "create"; + log.error(String.format("Failed to %s IOCs and threat intel source config", action), e); + lockService.releaseLock(lock, ActionListener.wrap( + r -> { + log.debug("Released threat intel source config lock with id [{}]", lock.getLockId()); + listener.onFailure(e); + }, + ex -> { + log.error(String.format("Unexpected failure while trying to release lock [%s] for tif source config.", lock.getLockId()), ex); + listener.onFailure(e); + } + )); + } + }, exception -> { + String action = RestRequest.Method.PUT.equals(request.getMethod()) ? "update" : "create"; + log.error(String.format("Failed to acquire lock while trying to %s tif source config", action), exception); + listener.onFailure(exception); + })); + } catch (Exception e) { + log.error("Failed to acquire lock for job", e); + listener.onFailure(e); + } + } +} + diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportListIOCsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportListIOCsAction.java new file mode 100644 index 000000000..80a6b538c --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportListIOCsAction.java @@ -0,0 +1,370 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.ActionRunnable; +import org.opensearch.action.StepListener; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.client.Client; +import org.opensearch.cluster.routing.Preference; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.SearchHit; +import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.Aggregations; +import org.opensearch.search.aggregations.bucket.terms.Terms; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.sort.FieldSortBuilder; +import org.opensearch.search.sort.SortBuilder; +import org.opensearch.search.sort.SortBuilders; +import org.opensearch.search.sort.SortOrder; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsAction; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsActionRequest; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsActionResponse; +import org.opensearch.securityanalytics.model.DetailedSTIX2IOCDto; +import org.opensearch.securityanalytics.model.STIX2IOC; +import org.opensearch.securityanalytics.model.STIX2IOCDto; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService; +import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.service.DefaultTifSourceConfigLoaderService; +import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; +import org.opensearch.securityanalytics.transport.SecureTransportAction; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.opensearch.securityanalytics.threatIntel.common.TIFJobState.AVAILABLE; +import static org.opensearch.securityanalytics.threatIntel.common.TIFJobState.REFRESHING; +import static org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService.getStateFieldName; + +public class TransportListIOCsAction extends HandledTransportAction implements SecureTransportAction { + private static final Logger log = LogManager.getLogger(TransportListIOCsAction.class); + + private final ClusterService clusterService; + private final TransportSearchTIFSourceConfigsAction transportSearchTIFSourceConfigsAction; + private final DefaultTifSourceConfigLoaderService defaultTifSourceConfigLoaderService; + private final Client client; + private final NamedXContentRegistry xContentRegistry; + private final ThreadPool threadPool; + private final SATIFSourceConfigService saTifSourceConfigService; + private final Settings settings; + private volatile Boolean filterByEnabled; + private final IocFindingService iocFindingService; + + public static String IOC_COUNT_AGG_NAME = "ioc_id_count"; + public static String IOC_ID_KEYWORD_FIELD = "ioc_feed_ids.ioc_id.keyword"; + + @Inject + public TransportListIOCsAction( + final ClusterService clusterService, + TransportService transportService, + TransportSearchTIFSourceConfigsAction transportSearchTIFSourceConfigsAction, + SATIFSourceConfigService saTifSourceConfigService, + DefaultTifSourceConfigLoaderService defaultTifSourceConfigLoaderService, + Client client, + NamedXContentRegistry xContentRegistry, + ActionFilters actionFilters, + Settings settings + ) { + super(ListIOCsAction.NAME, transportService, actionFilters, ListIOCsActionRequest::new); + this.clusterService = clusterService; + this.transportSearchTIFSourceConfigsAction = transportSearchTIFSourceConfigsAction; + this.saTifSourceConfigService = saTifSourceConfigService; + this.defaultTifSourceConfigLoaderService = defaultTifSourceConfigLoaderService; + this.client = client; + this.xContentRegistry = xContentRegistry; + this.threadPool = this.client.threadPool(); + this.settings = settings; + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.iocFindingService = new IocFindingService(client, clusterService, xContentRegistry); + } + + @Override + protected void doExecute(Task task, ListIOCsActionRequest request, ActionListener listener) { + AsyncListIOCsAction asyncAction = new AsyncListIOCsAction(task, request, listener); + asyncAction.start(); + } + + class AsyncListIOCsAction { + private ListIOCsActionRequest request; + private ActionListener listener; + + private final AtomicReference response; + private final AtomicBoolean counter = new AtomicBoolean(); + private final Task task; + + AsyncListIOCsAction(Task task, ListIOCsActionRequest request, ActionListener listener) { + this.task = task; + this.request = request; + this.listener = listener; + this.response = new AtomicReference<>(); + } + + void start() { + // validate user + User user = readUserFromThreadContext(TransportListIOCsAction.this.threadPool); + String validateBackendRoleMessage = validateUserBackendRoles(user, TransportListIOCsAction.this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + listener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); + return; + } + TransportListIOCsAction.this.threadPool.getThreadContext().stashContext(); // stash context to make calls as admin client + + StepListener defaultTifConfigsLoadedListener = null; + try { + defaultTifConfigsLoadedListener = new StepListener<>(); + defaultTifSourceConfigLoaderService.createDefaultTifConfigsIfNotExists(defaultTifConfigsLoadedListener); + defaultTifConfigsLoadedListener.whenComplete(r -> searchIocs(), e -> searchIocs()); + } catch (Exception e) { + log.error("Failed to load default tif source configs. Moving on to list iocs", e); + searchIocs(); + } + } + + private void searchIocs() { + /** get all match threat intel source configs. fetch write index of each config if no iocs provided else fetch just index alias */ + List configIds = request.getFeedIds() == null ? Collections.emptyList() : request.getFeedIds(); + saTifSourceConfigService.searchTIFSourceConfigs(getFeedsSearchSourceBuilder(configIds), + ActionListener.wrap( + searchResponse -> { + List iocIndices = new ArrayList<>(); + for (SearchHit hit : searchResponse.getHits().getHits()) { + XContentParser xcp = XContentType.JSON.xContent().createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString() + ); + SATIFSourceConfig config = SATIFSourceConfig.docParse(xcp, hit.getId(), hit.getVersion()); + if (config.getIocStoreConfig() instanceof DefaultIocStoreConfig) { + DefaultIocStoreConfig iocStoreConfig = (DefaultIocStoreConfig) config.getIocStoreConfig(); + for (DefaultIocStoreConfig.IocToIndexDetails iocToindexDetails : iocStoreConfig.getIocToIndexDetails()) { + String writeIndex = iocToindexDetails.getActiveIndex(); + if (writeIndex != null) { + iocIndices.add(writeIndex); + } + } + } + } + if (iocIndices.isEmpty()) { + log.info("No ioc indices found to query for given threat intel source filtering criteria {}", String.join(",", configIds)); + listener.onResponse(new ListIOCsActionResponse(0L, Collections.emptyList())); + return; + } + listIocs(iocIndices); + }, e -> { + log.error(String.format("Failed to fetch threat intel source configs. Unable to return Iocs"), e); + listener.onFailure(e); + } + )); + } + + private void listIocs(List iocIndices) { + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + + QueryBuilder typeQueryBuilder = QueryBuilders.boolQuery(); + + // If any of the 'type' options are 'ALL', do not apply 'type' filter + if (request.getTypes() != null && request.getTypes().stream().noneMatch(type -> ListIOCsActionRequest.ALL_TYPES_FILTER.equalsIgnoreCase(type))) { + for (String type : request.getTypes()) { + boolQueryBuilder.should(QueryBuilders.matchQuery(STIX2IOC.TYPE_FIELD, type)); + } + boolQueryBuilder.must(typeQueryBuilder); + } + + if (request.getTable().getSearchString() != null && !request.getTable().getSearchString().isEmpty()) { + boolQueryBuilder.must(QueryBuilders.queryStringQuery(request.getTable().getSearchString())); + } + + + SortBuilder sortBuilder = SortBuilders + .fieldSort(request.getTable().getSortString()) + .order(SortOrder.fromString(request.getTable().getSortOrder())); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .version(true) + .seqNoAndPrimaryTerm(true) + .fetchSource(true) + .trackTotalHits(true) + .query(boolQueryBuilder) + .sort(sortBuilder) + .size(request.getTable().getSize()) + .from(request.getTable().getStartIndex()); + + SearchRequest searchRequest = new SearchRequest() + .indices(iocIndices.toArray(new String[0])) + .source(searchSourceBuilder) + .preference(Preference.PRIMARY_FIRST.type()); + + client.search(searchRequest, new ActionListener<>() { + @Override + public void onResponse(SearchResponse searchResponse) { + if (searchResponse.isTimedOut()) { + onFailures(new OpenSearchStatusException("Search request timed out", RestStatus.REQUEST_TIMEOUT)); + } + + getFindingsCount(searchResponse); + } + + @Override + public void onFailure(Exception e) { + if (e instanceof IndexNotFoundException) { + // If no IOC system indexes are found, return empty list response + listener.onResponse(ListIOCsActionResponse.EMPTY_RESPONSE); + } else { + log.error("Failed to list IOCs.", e); + listener.onFailure(SecurityAnalyticsException.wrap(e)); + } + } + }); + } + + private void getFindingsCount(SearchResponse iocSearchResponse) { + // Concurrently compiling a separate list of IOC IDs to create the subsequent findings count searchRequest + Set iocIds = new HashSet<>(); + List iocs = new ArrayList<>(); + Arrays.stream(iocSearchResponse.getHits().getHits()) + .forEach(hit -> { + try { + XContentParser xcp = XContentType.JSON.xContent().createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, + hit.getSourceAsString()); + xcp.nextToken(); + + STIX2IOCDto ioc = STIX2IOCDto.parse(xcp, hit.getId(), hit.getVersion()); + + iocIds.add(ioc.getId()); + iocs.add(ioc); + } catch (Exception e) { + log.error( + () -> new ParameterizedMessage("Failed to parse IOC doc from hit {}", hit.getId()), e + ); + } + }); + + // Create an aggregation query that will group by the IOC IDs in the findings + SearchSourceBuilder findingsCountSourceBuilder = new SearchSourceBuilder() + .fetchSource(false) + .trackTotalHits(true) + .query(QueryBuilders.termsQuery(IOC_ID_KEYWORD_FIELD, iocIds)) + .size(0) + .aggregation( + AggregationBuilders + .terms(IOC_COUNT_AGG_NAME) + .field(IOC_ID_KEYWORD_FIELD) + .size(iocIds.size()) + ); + + iocFindingService.search(findingsCountSourceBuilder, new ActionListener<>() { + @Override + public void onResponse(SearchResponse findingsSearchResponse) { + Map iocIdToNumFindings = new HashMap<>(); + + // Retrieve and store the counts from the aggregation response + Aggregations aggregations = findingsSearchResponse.getAggregations(); + if (aggregations != null) { + Terms iocIdCount = aggregations.get(IOC_COUNT_AGG_NAME); + if (iocIdCount != null) { + for (Terms.Bucket bucket : iocIdCount.getBuckets()) { + String iocId = bucket.getKeyAsString(); + long findingCount = bucket.getDocCount(); + iocIdToNumFindings.put(iocId, (int) findingCount); + } + } + } + + // Iterate through each IOC returned by the SearchRequest to create the detailed model for response + List iocDetails = new ArrayList<>(); + iocs.forEach((ioc) -> { + Integer numFindings = iocIdToNumFindings.getOrDefault(ioc.getId(), 0); + iocDetails.add(new DetailedSTIX2IOCDto(ioc, numFindings)); + }); + + // Return API response + onOperation(new ListIOCsActionResponse(iocSearchResponse.getHits().getTotalHits().value, iocDetails)); + } + + @Override + public void onFailure(Exception e) { + log.error("Failed to get IOC findings count:", e); + listener.onFailure(SecurityAnalyticsException.wrap(e)); + } + }); + } + private void onOperation(ListIOCsActionResponse response) { + this.response.set(response); + if (counter.compareAndSet(false, true)) { + finishHim(response, null); + } + } + + private void onFailures(Exception t) { + if (counter.compareAndSet(false, true)) { + finishHim(null, t); + } + } + + private void finishHim(ListIOCsActionResponse response, Exception t) { + threadPool.executor(ThreadPool.Names.GENERIC).execute(ActionRunnable.supply(listener, () -> { + if (t != null) { + if (t instanceof OpenSearchStatusException) { + throw t; + } + throw SecurityAnalyticsException.wrap(t); + } else { + return response; + } + })); + } + } + + private SearchSourceBuilder getFeedsSearchSourceBuilder(List configIds) { + if (false == configIds.isEmpty()) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + for (String configId : configIds) { + queryBuilder.should(QueryBuilders.matchQuery("_id", configId)); + } + return new SearchSourceBuilder().query(queryBuilder).size(9999); + } else { + BoolQueryBuilder stateQueryBuilder = QueryBuilders.boolQuery() + .should(QueryBuilders.matchQuery(getStateFieldName(), REFRESHING.toString())) + .should(QueryBuilders.matchQuery(getStateFieldName(), AVAILABLE.toString())); + return new SearchSourceBuilder().query(stateQueryBuilder).size(9999); + } + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java similarity index 62% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobAction.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java index a50beda35..4c5bd5e4d 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel.action; +package org.opensearch.securityanalytics.threatIntel.transport; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -15,15 +15,22 @@ import org.opensearch.action.support.HandledTransportAction; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.commons.authuser.User; import org.opensearch.core.action.ActionListener; import org.opensearch.core.rest.RestStatus; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.jobscheduler.spi.LockModel; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction; +import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobRequest; +import org.opensearch.securityanalytics.threatIntel.action.ThreatIntelIndicesResponse; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobUpdateService; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService; +import org.opensearch.securityanalytics.transport.SecureTransportAction; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -37,13 +44,16 @@ /** * Transport action to create job to fetch threat intel feed data and save IoCs */ -public class TransportPutTIFJobAction extends HandledTransportAction { +public class TransportPutTIFJobAction extends HandledTransportAction implements SecureTransportAction { // TODO refactor this into a service class that creates feed updation job. This is not necessary to be a transport action private static final Logger log = LogManager.getLogger(TransportPutTIFJobAction.class); private final TIFJobParameterService tifJobParameterService; private final TIFJobUpdateService tifJobUpdateService; private final TIFLockService lockService; + private final Settings settings; + private final ThreadPool threadPool; + private volatile Boolean filterByEnabled; /** * Default constructor @@ -61,16 +71,29 @@ public TransportPutTIFJobAction( final ThreadPool threadPool, final TIFJobParameterService tifJobParameterService, final TIFJobUpdateService tifJobUpdateService, - final TIFLockService lockService + final TIFLockService lockService, + Settings settings ) { super(PutTIFJobAction.NAME, transportService, actionFilters, PutTIFJobRequest::new); this.tifJobParameterService = tifJobParameterService; this.tifJobUpdateService = tifJobUpdateService; this.lockService = lockService; + this.threadPool = threadPool; + this.settings = settings; + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); } @Override protected void doExecute(final Task task, final PutTIFJobRequest request, final ActionListener listener) { + User user = readUserFromThreadContext(this.threadPool); + + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + listener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); + return; + } + this.threadPool.getThreadContext().stashContext(); + try { lockService.acquireLock(request.getName(), LOCK_DURATION_IN_SECONDS, ActionListener.wrap(lock -> { if (lock == null) { @@ -83,9 +106,17 @@ protected void doExecute(final Task task, final PutTIFJobRequest request, final try { internalDoExecute(request, lock, listener); } catch (Exception e) { - lockService.releaseLock(lock); - listener.onFailure(e); - log.error("listener failed when executing", e); + log.error("Failed execution to put tif job action", e); + lockService.releaseLock(lock, ActionListener.wrap( + r -> { + log.debug("Released tif job parameter lock with id [{}]", lock.getLockId()); + listener.onFailure(e); + }, + ex -> { + log.error(String.format("Unexpected failure while trying to release lock [%s] for tif job parameter [%s].", lock.getLockId(), request.getName()), ex); + listener.onFailure(e); + } + )); } }, exception -> { listener.onFailure(exception); @@ -115,12 +146,19 @@ protected void internalDoExecute( listener.onFailure(e); } }, exception -> { - lockService.releaseLock(lock); - log.error("failed to release lock", exception); - listener.onFailure(exception); + log.error("Failed to save tif job parameter", exception); + lockService.releaseLock(lock, ActionListener.wrap( + r -> { + log.debug("Released tif job parameter lock with id [{}]", lock.getLockId()); + listener.onFailure(exception); + }, + ex -> { + log.error(String.format("Unexpected failure while trying to release lock [%s] for tif job parameter [%s].", lock.getLockId(), request.getName()), ex); + listener.onFailure(exception); + } + )); }); tifJobParameterService.createJobIndexIfNotExists(createIndexStepListener); - } /** @@ -138,22 +176,40 @@ protected ActionListener postIndexingTifJobParameter( createThreatIntelFeedData(tifJobParameter, lockService.getRenewLockRunnable(lockReference), ActionListener.wrap( threatIntelIndicesResponse -> { if (threatIntelIndicesResponse.isAcknowledged()) { - lockService.releaseLock(lockReference.get()); - listener.onResponse(new AcknowledgedResponse(true)); + lockService.releaseLock(lockReference.get(), ActionListener.wrap( + r -> { + log.debug("Released tif job parameter lock with id [{}]", lock.getLockId()); + listener.onResponse(new AcknowledgedResponse(true)); + }, + ex -> { + log.error(String.format("Unexpected failure while trying to release lock [%s] for tif job parameter [%s].", lock.getLockId(), tifJobParameter.getName()), ex); + listener.onFailure(ex); + } + )); } else { listener.onFailure(new OpenSearchStatusException("creation of threat intel feed data failed", RestStatus.INTERNAL_SERVER_ERROR)); } }, listener::onFailure )); }, e -> { - lockService.releaseLock(lock); + Exception exception; if (e instanceof VersionConflictEngineException) { log.error("tifJobParameter already exists"); - listener.onFailure(new ResourceAlreadyExistsException("tifJobParameter [{}] already exists", tifJobParameter.getName())); + exception = new ResourceAlreadyExistsException("tifJobParameter [{}] already exists", tifJobParameter.getName()); } else { log.error("Internal server error"); - listener.onFailure(e); + exception = e; } + lockService.releaseLock(lock, ActionListener.wrap( + r -> { + log.debug("Released tif job parameter lock with id [{}]", lock.getLockId()); + listener.onFailure(exception); + }, + ex -> { + log.error(String.format("Unexpected failure while trying to release lock [%s] for tif job parameter [%s].", lock.getLockId(), tifJobParameter.getName()), ex); + listener.onFailure(exception); + } + )); } ); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportRefreshTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportRefreshTIFSourceConfigAction.java new file mode 100644 index 000000000..0c8af386f --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportRefreshTIFSourceConfigAction.java @@ -0,0 +1,79 @@ +package org.opensearch.securityanalytics.threatIntel.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.SARefreshTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.SARefreshTIFSourceConfigRequest; +import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService; +import org.opensearch.securityanalytics.transport.SecureTransportAction; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +public class TransportRefreshTIFSourceConfigAction extends HandledTransportAction implements SecureTransportAction { + + private static final Logger log = LogManager.getLogger(TransportRefreshTIFSourceConfigAction.class); + + private final ClusterService clusterService; + + private final Settings settings; + + private final ThreadPool threadPool; + + private volatile Boolean filterByEnabled; + + private final SATIFSourceConfigManagementService saTifSourceConfigManagementService; + + @Inject + public TransportRefreshTIFSourceConfigAction(TransportService transportService, + ActionFilters actionFilters, + ClusterService clusterService, + final ThreadPool threadPool, + Settings settings, + final SATIFSourceConfigManagementService saTifSourceConfigManagementService) { + super(SARefreshTIFSourceConfigAction.NAME, transportService, actionFilters, SARefreshTIFSourceConfigRequest::new); + this.clusterService = clusterService; + this.threadPool = threadPool; + this.settings = settings; + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled); + this.saTifSourceConfigManagementService = saTifSourceConfigManagementService; + } + + @Override + protected void doExecute(Task task, SARefreshTIFSourceConfigRequest request, ActionListener actionListener) { + // validate user + User user = readUserFromThreadContext(this.threadPool); + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + actionListener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); + return; + } + this.threadPool.getThreadContext().stashContext(); + + saTifSourceConfigManagementService.refreshTIFSourceConfig(request.getId(), user, ActionListener.wrap( + r -> actionListener.onResponse( + new AcknowledgedResponse(true) + ), e -> { + log.error("Failed to refresh threat intel source config for [{}]", request.getId()); + actionListener.onFailure(e); + }) + ); + } + + private void setFilterByEnabled(boolean filterByEnabled) { + this.filterByEnabled = filterByEnabled; + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportSearchTIFSourceConfigsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportSearchTIFSourceConfigsAction.java new file mode 100644 index 000000000..877728e31 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportSearchTIFSourceConfigsAction.java @@ -0,0 +1,109 @@ +package org.opensearch.securityanalytics.threatIntel.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.StepListener; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.SASearchTIFSourceConfigsAction; +import org.opensearch.securityanalytics.threatIntel.action.SASearchTIFSourceConfigsRequest; +import org.opensearch.securityanalytics.threatIntel.service.DefaultTifSourceConfigLoaderService; +import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService; +import org.opensearch.securityanalytics.transport.SecureTransportAction; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +public class TransportSearchTIFSourceConfigsAction extends HandledTransportAction implements SecureTransportAction { + + private static final Logger log = LogManager.getLogger(TransportSearchTIFSourceConfigsAction.class); + + private final ClusterService clusterService; + + private final Settings settings; + private final DefaultTifSourceConfigLoaderService defaultTifSourceConfigLoaderService; + + private final ThreadPool threadPool; + + private volatile Boolean filterByEnabled; + + private final SATIFSourceConfigManagementService saTifConfigService; + + @Inject + public TransportSearchTIFSourceConfigsAction(TransportService transportService, + ActionFilters actionFilters, + ClusterService clusterService, + final ThreadPool threadPool, + Settings settings, + DefaultTifSourceConfigLoaderService defaultTifSourceConfigLoaderService, + final SATIFSourceConfigManagementService saTifConfigService) { + super(SASearchTIFSourceConfigsAction.NAME, transportService, actionFilters, SASearchTIFSourceConfigsRequest::new); + this.clusterService = clusterService; + this.threadPool = threadPool; + this.settings = settings; + this.defaultTifSourceConfigLoaderService = defaultTifSourceConfigLoaderService; + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled); + this.saTifConfigService = saTifConfigService; + } + + @Override + protected void doExecute(Task task, SASearchTIFSourceConfigsRequest request, ActionListener actionListener) { + // validate user + User user = readUserFromThreadContext(this.threadPool); + + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + actionListener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); + return; + } + this.threadPool.getThreadContext().stashContext(); // stash context to make calls as admin client + + StepListener defaultTifConfigsLoadedListener; + try { + defaultTifConfigsLoadedListener = new StepListener<>(); + defaultTifSourceConfigLoaderService.createDefaultTifConfigsIfNotExists(defaultTifConfigsLoadedListener); + defaultTifConfigsLoadedListener.whenComplete(res -> saTifConfigService.searchTIFSourceConfigs(request.getSearchSourceBuilder(), ActionListener.wrap( + r -> { + log.debug("Successfully listed all threat intel source configs"); + actionListener.onResponse(r); + }, e -> { + log.error("Failed to list all threat intel source configs"); + actionListener.onFailure(e); + } + )), ex -> saTifConfigService.searchTIFSourceConfigs(request.getSearchSourceBuilder(), ActionListener.wrap( + r -> { + log.debug("Successfully listed all threat intel source configs"); + actionListener.onResponse(r); + }, e -> { + log.error("Failed to list all threat intel source configs"); + actionListener.onFailure(e); + } + ))); + } catch (Exception e) { + log.error("Failed to load default tif source configs. Moving on to list iocs", e); + saTifConfigService.searchTIFSourceConfigs(request.getSearchSourceBuilder(), ActionListener.wrap( + r -> { + log.debug("Successfully listed all threat intel source configs"); + actionListener.onResponse(r); + }, ex -> { + log.error("Failed to list all threat intel source configs"); + actionListener.onFailure(e); + })); + } + } + + private void setFilterByEnabled(boolean filterByEnabled) { + this.filterByEnabled = filterByEnabled; + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportDeleteThreatIntelMonitorAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportDeleteThreatIntelMonitorAction.java new file mode 100644 index 000000000..1ecebc2e3 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportDeleteThreatIntelMonitorAction.java @@ -0,0 +1,70 @@ +package org.opensearch.securityanalytics.threatIntel.transport.monitor; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.Client; +import org.opensearch.client.node.NodeClient; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.commons.alerting.AlertingPluginInterface; +import org.opensearch.commons.alerting.action.DeleteMonitorRequest; +import org.opensearch.commons.alerting.action.DeleteMonitorResponse; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.monitor.DeleteThreatIntelMonitorAction; +import org.opensearch.securityanalytics.threatIntel.action.monitor.request.DeleteThreatIntelMonitorRequest; +import org.opensearch.securityanalytics.transport.SecureTransportAction; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +public class TransportDeleteThreatIntelMonitorAction extends HandledTransportAction implements SecureTransportAction { + + private static final Logger log = LogManager.getLogger(TransportDeleteThreatIntelMonitorAction.class); + + private final ThreadPool threadPool; + private final Settings settings; + private final NamedWriteableRegistry namedWriteableRegistry; + private final Client client; + private volatile Boolean filterByEnabled; + + @Inject + public TransportDeleteThreatIntelMonitorAction( + final TransportService transportService, + final ActionFilters actionFilters, + final ThreadPool threadPool, + final Settings settings, + final Client client, + final NamedWriteableRegistry namedWriteableRegistry + ) { + super(DeleteThreatIntelMonitorAction.NAME, transportService, actionFilters, DeleteThreatIntelMonitorRequest::new); + this.threadPool = threadPool; + this.settings = settings; + this.namedWriteableRegistry = namedWriteableRegistry; + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.client = client; + } + + @Override + protected void doExecute(Task task, DeleteThreatIntelMonitorRequest request, ActionListener listener) { + User user = readUserFromThreadContext(this.threadPool); + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(validateBackendRoleMessage, RestStatus.FORBIDDEN))); + return; + } + this.threadPool.getThreadContext().stashContext(); + + AlertingPluginInterface.INSTANCE.deleteMonitor((NodeClient) client, + new DeleteMonitorRequest(request.getMonitorId(), WriteRequest.RefreshPolicy.IMMEDIATE), + listener); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportGetThreatIntelAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportGetThreatIntelAlertsAction.java new file mode 100644 index 000000000..65feccc6f --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportGetThreatIntelAlertsAction.java @@ -0,0 +1,187 @@ +package org.opensearch.securityanalytics.threatIntel.transport.monitor; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.SearchHit; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.sort.FieldSortBuilder; +import org.opensearch.search.sort.SortBuilders; +import org.opensearch.search.sort.SortOrder; +import org.opensearch.securityanalytics.model.threatintel.ThreatIntelAlert; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.monitor.GetThreatIntelAlertsAction; +import org.opensearch.securityanalytics.threatIntel.action.monitor.request.GetThreatIntelAlertsRequest; +import org.opensearch.securityanalytics.threatIntel.action.monitor.request.SearchThreatIntelMonitorRequest; +import org.opensearch.securityanalytics.threatIntel.action.monitor.response.GetThreatIntelAlertsResponse; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.ThreatIntelAlertService; +import org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelMonitorRunner; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelAlertDto; +import org.opensearch.securityanalytics.transport.SecureTransportAction; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static org.opensearch.securityanalytics.transport.TransportIndexDetectorAction.PLUGIN_OWNER_FIELD; + +public class TransportGetThreatIntelAlertsAction extends HandledTransportAction implements SecureTransportAction { + + private final Client client; + private final TransportSearchThreatIntelMonitorAction transportSearchThreatIntelMonitorAction; + + private final NamedXContentRegistry xContentRegistry; + + private final ClusterService clusterService; + + private final Settings settings; + + private final ThreadPool threadPool; + + private final ThreatIntelAlertService alertsService; + + private volatile Boolean filterByEnabled; + + private static final Logger log = LogManager.getLogger(TransportGetThreatIntelAlertsAction.class); + + + @Inject + public TransportGetThreatIntelAlertsAction(TransportService transportService, + ActionFilters actionFilters, + ClusterService clusterService, + ThreadPool threadPool, + Settings settings, + NamedXContentRegistry xContentRegistry, + Client client, + TransportSearchThreatIntelMonitorAction transportSearchThreatIntelMonitorAction1, ThreatIntelAlertService alertsService) { + super(GetThreatIntelAlertsAction.NAME, transportService, actionFilters, GetThreatIntelAlertsRequest::new); + this.client = client; + this.transportSearchThreatIntelMonitorAction = transportSearchThreatIntelMonitorAction1; + this.xContentRegistry = xContentRegistry; + this.clusterService = clusterService; + this.threadPool = threadPool; + this.settings = settings; + this.alertsService = alertsService; + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled); + } + + private void setFilterByEnabled(boolean filterByEnabled) { + this.filterByEnabled = filterByEnabled; + } + + @Override + protected void doExecute(Task task, GetThreatIntelAlertsRequest request, ActionListener listener) { + User user = readUserFromThreadContext(this.threadPool); + + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + listener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); + return; + } + this.threadPool.getThreadContext().stashContext(); + + //fetch monitors and search + SearchRequest threatIntelMonitorsSearchRequest = new SearchRequest(); + threatIntelMonitorsSearchRequest.indices(".opendistro-alerting-config"); + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.must().add(new BoolQueryBuilder().must(QueryBuilders.matchPhraseQuery("monitor.owner", PLUGIN_OWNER_FIELD))); + boolQueryBuilder.must().add(new BoolQueryBuilder().must(QueryBuilders.matchPhraseQuery("monitor.monitor_type", ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE))); + threatIntelMonitorsSearchRequest.source(new SearchSourceBuilder().query(boolQueryBuilder)); + transportSearchThreatIntelMonitorAction.execute(new SearchThreatIntelMonitorRequest(threatIntelMonitorsSearchRequest), ActionListener.wrap( + searchResponse -> { + List monitorIds = searchResponse.getHits() == null || searchResponse.getHits().getHits() == null ? new ArrayList<>() : + Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getId).collect(Collectors.toList()); + if (monitorIds.isEmpty()) { + listener.onResponse(new GetThreatIntelAlertsResponse(Collections.emptyList(), 0)); + return; + } + getAlerts(monitorIds, request, listener); + }, + + e -> { + if (e instanceof IndexNotFoundException) { + log.debug("Monitor index not created. Returning 0 threat intel alerts"); + listener.onResponse(new GetThreatIntelAlertsResponse(Collections.emptyList(), 0)); + return; + } + log.error("Failed to get threat intel monitor alerts", e); + listener.onFailure(e); + } + )); + } + + private void getAlerts(List monitorIds, + GetThreatIntelAlertsRequest request, + ActionListener listener) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + BoolQueryBuilder monitorIdMatchQuery = QueryBuilders.boolQuery(); + for (String monitorId : monitorIds) { + monitorIdMatchQuery.should(QueryBuilders.boolQuery() + .must(QueryBuilders.matchPhraseQuery("monitor_id", monitorId))); + + } + queryBuilder.filter(monitorIdMatchQuery); + Table tableProp = request.getTable(); + FieldSortBuilder sortBuilder = SortBuilders + .fieldSort(tableProp.getSortString()) + .order(SortOrder.fromString(tableProp.getSortOrder())); + if (tableProp.getMissing() != null && !tableProp.getMissing().isEmpty()) { + sortBuilder.missing(tableProp.getMissing()); + } + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .version(true) + .seqNoAndPrimaryTerm(true) + .query(queryBuilder) + .sort(sortBuilder) + .size(tableProp.getSize()) + .from(tableProp.getStartIndex()); + alertsService.search(searchSourceBuilder, ActionListener.wrap( + searchResponse -> { + List alerts = new ArrayList<>(); + if (searchResponse.getHits() == null || searchResponse.getHits().getHits() == null || searchResponse.getHits().getHits().length == 0) { + listener.onResponse(new GetThreatIntelAlertsResponse(Collections.emptyList(), 0)); + return; + } + for (SearchHit hit : searchResponse.getHits().getHits()) { + XContentParser xcp = XContentType.JSON.xContent().createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString() + ); + if (xcp.currentToken() == null) + xcp.nextToken(); + ThreatIntelAlert alert = ThreatIntelAlert.parse(xcp, hit.getVersion()); + alerts.add(new ThreatIntelAlertDto(alert, hit.getSeqNo(), hit.getPrimaryTerm())); + } + listener.onResponse(new GetThreatIntelAlertsResponse(alerts, (int) searchResponse.getHits().getTotalHits().value)); + }, e -> { + log.error("Failed to search for threat intel alerts", e); + listener.onFailure(e); + } + )); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java new file mode 100644 index 000000000..c4902b99b --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java @@ -0,0 +1,257 @@ +package org.opensearch.securityanalytics.threatIntel.transport.monitor; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.OpenSearchException; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.ResourceAlreadyExistsException; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.Client; +import org.opensearch.client.node.NodeClient; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.commons.alerting.AlertingPluginInterface; +import org.opensearch.commons.alerting.action.IndexMonitorRequest; +import org.opensearch.commons.alerting.action.IndexMonitorResponse; +import org.opensearch.commons.alerting.model.DataSources; +import org.opensearch.commons.alerting.model.DocLevelMonitorInput; +import org.opensearch.commons.alerting.model.Monitor; +import org.opensearch.commons.alerting.model.remote.monitors.RemoteDocLevelMonitorInput; +import org.opensearch.commons.alerting.model.remote.monitors.RemoteMonitorTrigger; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.seqno.SequenceNumbers; +import org.opensearch.rest.RestRequest; +import org.opensearch.search.SearchHit; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.monitor.IndexThreatIntelMonitorAction; +import org.opensearch.securityanalytics.threatIntel.action.monitor.request.IndexThreatIntelMonitorRequest; +import org.opensearch.securityanalytics.threatIntel.action.monitor.request.SearchThreatIntelMonitorRequest; +import org.opensearch.securityanalytics.threatIntel.action.monitor.response.IndexThreatIntelMonitorResponse; +import org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelMonitorRunner; +import org.opensearch.securityanalytics.threatIntel.model.monitor.PerIocTypeScanInput; +import org.opensearch.securityanalytics.threatIntel.model.monitor.ThreatIntelInput; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelTriggerDto; +import org.opensearch.securityanalytics.threatIntel.util.ThreatIntelMonitorUtils; +import org.opensearch.securityanalytics.transport.SecureTransportAction; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE; +import static org.opensearch.securityanalytics.transport.TransportIndexDetectorAction.PLUGIN_OWNER_FIELD; + +public class TransportIndexThreatIntelMonitorAction extends HandledTransportAction implements SecureTransportAction { + private static final Logger log = LogManager.getLogger(TransportIndexThreatIntelMonitorAction.class); + + private final TransportSearchThreatIntelMonitorAction transportSearchThreatIntelMonitorAction; + private final ThreadPool threadPool; + private final Settings settings; + private final NamedWriteableRegistry namedWriteableRegistry; + private final NamedXContentRegistry xContentRegistry; + private final Client client; + private volatile Boolean filterByEnabled; + private final TimeValue indexTimeout; + + @Inject + public TransportIndexThreatIntelMonitorAction( + final TransportService transportService, + final TransportSearchThreatIntelMonitorAction transportSearchThreatIntelMonitorAction, + final ActionFilters actionFilters, + final ThreadPool threadPool, + final Settings settings, + final Client client, + final NamedWriteableRegistry namedWriteableRegistry, + final NamedXContentRegistry namedXContentRegistry + ) { + super(IndexThreatIntelMonitorAction.NAME, transportService, actionFilters, IndexThreatIntelMonitorRequest::new); + this.transportSearchThreatIntelMonitorAction = transportSearchThreatIntelMonitorAction; + this.threadPool = threadPool; + this.settings = settings; + this.namedWriteableRegistry = namedWriteableRegistry; + this.xContentRegistry = namedXContentRegistry; + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.indexTimeout = SecurityAnalyticsSettings.INDEX_TIMEOUT.get(this.settings); + this.client = client; + } + + @Override + protected void doExecute(Task task, IndexThreatIntelMonitorRequest request, ActionListener listener) { + try { + // validate user + User user = readUserFromThreadContext(this.threadPool); + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(validateBackendRoleMessage, RestStatus.FORBIDDEN))); + return; + } + this.threadPool.getThreadContext().stashContext(); + + if(request.getMethod().equals(RestRequest.Method.PUT)) { + indexMonitor(request, listener, user); + return; + } + + //fetch monitors and search to ensure only one threat intel monitor can be created + SearchRequest threatIntelMonitorsSearchRequest = new SearchRequest(); + threatIntelMonitorsSearchRequest.indices(".opendistro-alerting-config"); + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.must().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.owner", PLUGIN_OWNER_FIELD))); + boolQueryBuilder.must().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.monitor_type", ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE))); + threatIntelMonitorsSearchRequest.source(new SearchSourceBuilder().query(boolQueryBuilder)); + transportSearchThreatIntelMonitorAction.execute(new SearchThreatIntelMonitorRequest(threatIntelMonitorsSearchRequest), ActionListener.wrap( + searchResponse -> { + List monitorIds = searchResponse.getHits() == null || searchResponse.getHits().getHits() == null ? new ArrayList<>() : + Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getId).collect(Collectors.toList()); + if (monitorIds.isEmpty()) { + indexMonitor(request, listener, user); + } else + listener.onFailure(new ResourceAlreadyExistsException(String.format("Threat intel monitor %s already exists.", monitorIds.get(0)))); + }, + + e -> { + if (e instanceof IndexNotFoundException || e.getMessage().contains("Configured indices are not found")) { + try { + indexMonitor(request, listener, user); + return; + } catch (IOException ex) { + log.error(() -> new ParameterizedMessage("Unexpected failure while indexing threat intel monitor {} named {}", request.getId(), request.getMonitor().getName())); + listener.onFailure(new SecurityAnalyticsException("Unexpected failure while indexing threat intel monitor", RestStatus.INTERNAL_SERVER_ERROR, e)); + return; + } + } + log.error("Failed to update threat intel monitor alerts status", e); + listener.onFailure(e); + } + )); + + } catch (OpenSearchException e) { + log.error(() -> new ParameterizedMessage("Unexpected failure while indexing threat intel monitor {} named {}", request.getId(), request.getMonitor().getName())); + listener.onFailure(new SecurityAnalyticsException("Unexpected failure while indexing threat intel monitor", e.status(), e)); + } catch (Exception e) { + log.error(() -> new ParameterizedMessage("Unexpected failure while indexing threat intel monitor {} named {}", request.getId(), request.getMonitor().getName())); + listener.onFailure(new SecurityAnalyticsException("Unexpected failure while indexing threat intel monitor", RestStatus.INTERNAL_SERVER_ERROR, e)); + } + } + + private void indexMonitor(IndexThreatIntelMonitorRequest request, ActionListener listener, User user) throws IOException { + IndexMonitorRequest indexMonitorRequest = buildIndexMonitorRequest(request); + AlertingPluginInterface.INSTANCE.indexMonitor((NodeClient) client, indexMonitorRequest, namedWriteableRegistry, ActionListener.wrap( + r -> { + log.debug( + "{} threat intel monitor {}", request.getMethod() == RestRequest.Method.PUT ? "Updated" : "Created", + r.getId() + ); + IndexThreatIntelMonitorResponse response = getIndexThreatIntelMonitorResponse(r, user); + listener.onResponse(response); + }, e -> { + String errorText = "Failed to create threat intel monitor"; + SecurityAnalyticsException exception = new SecurityAnalyticsException(errorText, RestStatus.INTERNAL_SERVER_ERROR, e); + log.error(errorText, e); + if (e instanceof OpenSearchException) { + exception = new SecurityAnalyticsException(errorText, ((OpenSearchException) e).status(), e); + } + listener.onFailure(exception); + } + )); + } + + private IndexThreatIntelMonitorResponse getIndexThreatIntelMonitorResponse(IndexMonitorResponse r, User user) throws IOException { + IndexThreatIntelMonitorResponse response = new IndexThreatIntelMonitorResponse(r.getId(), r.getVersion(), r.getSeqNo(), r.getPrimaryTerm(), + ThreatIntelMonitorUtils.buildThreatIntelMonitorDto(r.getId(), r.getMonitor(), xContentRegistry)); + return response; + } + + private IndexMonitorRequest buildIndexMonitorRequest(IndexThreatIntelMonitorRequest request) throws IOException { + String id = request.getMethod() == RestRequest.Method.POST ? Monitor.NO_ID : request.getId(); + return new IndexMonitorRequest( + id, + SequenceNumbers.UNASSIGNED_SEQ_NO, + SequenceNumbers.UNASSIGNED_PRIMARY_TERM, + WriteRequest.RefreshPolicy.IMMEDIATE, + request.getMethod(), + buildThreatIntelMonitor(request), + null + ); + } + + private Monitor buildThreatIntelMonitor(IndexThreatIntelMonitorRequest request) throws IOException { + //TODO replace with threat intel monitor + DocLevelMonitorInput docLevelMonitorInput = new DocLevelMonitorInput( + String.format("threat intel input for monitor named %s", request.getMonitor().getName()), + request.getMonitor().getIndices(), + Collections.emptyList(), // no percolate queries + true + ); + List perIocTypeScanInputs = request.getMonitor().getPerIocTypeScanInputList().stream().map( + it -> new PerIocTypeScanInput(it.getIocType(), it.getIndexToFieldsMap()) + ).collect(Collectors.toList()); + ThreatIntelInput threatIntelInput = new ThreatIntelInput(perIocTypeScanInputs); + RemoteDocLevelMonitorInput remoteDocLevelMonitorInput = new RemoteDocLevelMonitorInput( + threatIntelInput.getThreatIntelInputAsBytesReference(), + docLevelMonitorInput); + List triggers = new ArrayList<>(); + for (ThreatIntelTriggerDto it : request.getMonitor().getTriggers()) { + try { + RemoteMonitorTrigger trigger = ThreatIntelMonitorUtils.buildRemoteMonitorTrigger(it); + triggers.add(trigger); + } catch (IOException e) { + logger.error(() -> new ParameterizedMessage("failed to parse threat intel trigger {}", it.getId()), e); + throw new RuntimeException(e); + } + } + + Monitor monitor; + try { + monitor = new Monitor( + request.getMethod() == RestRequest.Method.POST ? Monitor.NO_ID : request.getId(), + Monitor.NO_VERSION, + StringUtils.isBlank(request.getMonitor().getName()) ? "threat_intel_monitor" : request.getMonitor().getName(), + request.getMonitor().isEnabled(), + request.getMonitor().getSchedule(), + Instant.now(), + request.getMonitor().isEnabled() ? Instant.now() : null, + THREAT_INTEL_MONITOR_TYPE, + request.getMonitor().getUser(), + 1, + List.of(remoteDocLevelMonitorInput), + triggers, + Collections.emptyMap(), + new DataSources(), + false, + null, + PLUGIN_OWNER_FIELD + ); + } catch (Exception e) { + String error = "Error occurred while parsing monitor."; + log.error(error, e); + throw new SecurityAnalyticsException(error, RestStatus.BAD_REQUEST, e); + } + return monitor; + } + + +} + diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportSearchThreatIntelMonitorAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportSearchThreatIntelMonitorAction.java new file mode 100644 index 000000000..11a1e1beb --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportSearchThreatIntelMonitorAction.java @@ -0,0 +1,110 @@ +package org.opensearch.securityanalytics.threatIntel.transport.monitor; + +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.client.Client; +import org.opensearch.client.node.NodeClient; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.commons.alerting.AlertingPluginInterface; +import org.opensearch.commons.alerting.action.SearchMonitorRequest; +import org.opensearch.commons.alerting.model.Monitor; +import org.opensearch.commons.alerting.model.ScheduledJob; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.search.SearchHit; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.monitor.SearchThreatIntelMonitorAction; +import org.opensearch.securityanalytics.threatIntel.action.monitor.request.SearchThreatIntelMonitorRequest; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto; +import org.opensearch.securityanalytics.threatIntel.util.ThreatIntelMonitorUtils; +import org.opensearch.securityanalytics.transport.SecureTransportAction; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import static org.opensearch.securityanalytics.util.DetectorUtils.getEmptySearchResponse; + +public class TransportSearchThreatIntelMonitorAction extends HandledTransportAction implements SecureTransportAction { + + private final NamedXContentRegistry xContentRegistry; + private final Client client; + private final ClusterService clusterService; + private final Settings settings; + private final ThreadPool threadPool; + private Boolean filterByEnabled; + + @Inject + public TransportSearchThreatIntelMonitorAction(TransportService transportService, + ClusterService clusterService, + ActionFilters actionFilters, + NamedXContentRegistry xContentRegistry, + Settings settings, + Client client, + ThreadPool threadPool) { + super(SearchThreatIntelMonitorAction.NAME, transportService, actionFilters, SearchThreatIntelMonitorRequest::new); + this.xContentRegistry = xContentRegistry; + this.client = client; + this.clusterService = clusterService; + this.settings = settings; + this.threadPool = threadPool; + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled); + } + + @Override + protected void doExecute(Task task, SearchThreatIntelMonitorRequest request, ActionListener listener) { + User user = readUserFromThreadContext(this.threadPool); + +// if (doFilterForUser(user, this.filterByEnabled)) { +// // security is enabled and filterby is enabled +// log.info("Filtering result by: {}", user.getBackendRoles()); +// addFilter(user, request.searchRequest().source(), "detector.user.backend_roles.keyword"); +// } // TODO + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + listener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); + return; + } + this.threadPool.getThreadContext().stashContext(); + + //TODO change search request to fetch threat intel monitors + AlertingPluginInterface.INSTANCE.searchMonitors((NodeClient) client, new SearchMonitorRequest(request.searchRequest()), ActionListener.wrap( + response -> { + for (SearchHit hit : response.getHits().getHits()) { + XContentParser parser = XContentType.JSON.xContent() + .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString()); + ScheduledJob monitor = ScheduledJob.Companion.parse(parser, hit.getId(), hit.getVersion()); + ThreatIntelMonitorDto threatIntelMonitorDto = ThreatIntelMonitorUtils.buildThreatIntelMonitorDto(hit.getId(), (Monitor) monitor, xContentRegistry); + XContentBuilder builder = threatIntelMonitorDto.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), null); + hit.sourceRef(BytesReference.bytes(builder)); + } + listener.onResponse(response); + }, + e -> { + if (e instanceof IndexNotFoundException) { + listener.onResponse(getEmptySearchResponse()); + return; + } + log.error("Failed to search threat intel monitors", e); + listener.onFailure(e); + } + )); + } + + + private void setFilterByEnabled(boolean filterByEnabled) { + this.filterByEnabled = filterByEnabled; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportUpdateThreatIntelAlertStatusAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportUpdateThreatIntelAlertStatusAction.java new file mode 100644 index 000000000..024a8666b --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportUpdateThreatIntelAlertStatusAction.java @@ -0,0 +1,309 @@ +package org.opensearch.securityanalytics.threatIntel.transport.monitor; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.ResourceNotFoundException; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.commons.alerting.model.Alert; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.SearchHit; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.model.threatintel.ThreatIntelAlert; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.monitor.UpdateThreatIntelAlertStatusAction; +import org.opensearch.securityanalytics.threatIntel.action.monitor.request.SearchThreatIntelMonitorRequest; +import org.opensearch.securityanalytics.threatIntel.action.monitor.request.UpdateThreatIntelAlertStatusRequest; +import org.opensearch.securityanalytics.threatIntel.action.monitor.response.UpdateThreatIntelAlertsStatusResponse; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.ThreatIntelAlertService; +import org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelMonitorRunner; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelAlertDto; +import org.opensearch.securityanalytics.transport.SecureTransportAction; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static java.util.Collections.emptyList; +import static org.opensearch.securityanalytics.transport.TransportIndexDetectorAction.PLUGIN_OWNER_FIELD; + +public class TransportUpdateThreatIntelAlertStatusAction extends HandledTransportAction implements SecureTransportAction { + private final Client client; + private final TransportSearchThreatIntelMonitorAction transportSearchThreatIntelMonitorAction; + + private final NamedXContentRegistry xContentRegistry; + + private final ClusterService clusterService; + + private final Settings settings; + + private final ThreadPool threadPool; + + private final ThreatIntelAlertService alertsService; + + private volatile Boolean filterByEnabled; + + private static final Logger log = LogManager.getLogger(TransportUpdateThreatIntelAlertStatusAction.class); + + + @Inject + public TransportUpdateThreatIntelAlertStatusAction(TransportService transportService, + ActionFilters actionFilters, + ClusterService clusterService, + ThreadPool threadPool, + Settings settings, + NamedXContentRegistry xContentRegistry, + Client client, + TransportSearchThreatIntelMonitorAction transportSearchThreatIntelMonitorAction1, ThreatIntelAlertService alertsService) { + super(UpdateThreatIntelAlertStatusAction.NAME, transportService, actionFilters, UpdateThreatIntelAlertStatusRequest::new); + this.client = client; + this.transportSearchThreatIntelMonitorAction = transportSearchThreatIntelMonitorAction1; + this.xContentRegistry = xContentRegistry; + this.clusterService = clusterService; + this.threadPool = threadPool; + this.settings = settings; + this.alertsService = alertsService; + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled); + } + + @Override + protected void doExecute(Task task, UpdateThreatIntelAlertStatusRequest request, ActionListener listener) { + User user = readUserFromThreadContext(this.threadPool); + + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + listener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); + return; + } + this.threadPool.getThreadContext().stashContext(); + + //fetch monitors and search + SearchRequest threatIntelMonitorsSearchRequest = new SearchRequest(); + threatIntelMonitorsSearchRequest.indices(".opendistro-alerting-config"); + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.must().add(new BoolQueryBuilder().must(QueryBuilders.matchPhraseQuery("monitor.owner", PLUGIN_OWNER_FIELD))); + boolQueryBuilder.must().add(new BoolQueryBuilder().must(QueryBuilders.matchPhraseQuery("monitor.monitor_type", ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE))); + threatIntelMonitorsSearchRequest.source(new SearchSourceBuilder().query(boolQueryBuilder)); + transportSearchThreatIntelMonitorAction.execute(new SearchThreatIntelMonitorRequest(threatIntelMonitorsSearchRequest), ActionListener.wrap( + searchResponse -> { + List monitorIds = searchResponse.getHits() == null || searchResponse.getHits().getHits() == null ? new ArrayList<>() : + Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getId).collect(Collectors.toList()); + if (monitorIds.isEmpty()) { + log.error("Threat intel monitor not found. No alerts to update"); + listener.onFailure(new SecurityAnalyticsException("Threat intel monitor not found. No alerts to update", + RestStatus.BAD_REQUEST, + new IllegalArgumentException("Threat intel monitor not found. No alerts to update"))); + } + onSearchMonitorResponse(monitorIds, request, listener); + }, + + e -> { + if (e instanceof IndexNotFoundException) { + log.error("Threat intel monitor not found. No alerts to update"); + listener.onFailure(new SecurityAnalyticsException("Threat intel monitor not found. No alerts to update", + RestStatus.BAD_REQUEST, + new IllegalArgumentException("Threat intel monitor not found. No alerts to update"))); + return; + } + log.error("Failed to update threat intel monitor alerts status", e); + listener.onFailure(e); + } + )); + + } + + private void onSearchMonitorResponse(List monitorIds, + UpdateThreatIntelAlertStatusRequest request, + ActionListener listener) { + SearchSourceBuilder searchSourceBuilder = getSearchSourceQueryingForAlertsToUpdate(monitorIds, request, listener); + alertsService.search(searchSourceBuilder, ActionListener.wrap( + searchResponse -> { + List alerts = new ArrayList<>(); + if (searchResponse.getHits() == null || searchResponse.getHits().getHits() == null || searchResponse.getHits().getHits().length == 0) { + log.error("No alerts found to update"); + listener.onFailure(new SecurityAnalyticsException("No alerts found to update", + RestStatus.BAD_REQUEST, + new ResourceNotFoundException("No alerts found to update"))); + return; + } + for (SearchHit hit : searchResponse.getHits().getHits()) { + XContentParser xcp = XContentType.JSON.xContent().createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString() + ); + if (xcp.currentToken() == null) + xcp.nextToken(); + ThreatIntelAlert alert = ThreatIntelAlert.parse( + xcp, + hit.getVersion(), + hit.getSeqNo(), + hit.getPrimaryTerm() + ); + alerts.add(alert); + } + updateAlerts(monitorIds, alerts, request.getState(), listener); + }, e -> { + log.error("Failed to search for threat intel alerts", e); + listener.onFailure(e); + } + )); + } + + private static SearchSourceBuilder getSearchSourceQueryingForAlertsToUpdate(List monitorIds, UpdateThreatIntelAlertStatusRequest request, ActionListener listener) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + BoolQueryBuilder monitorIdMatchQuery = QueryBuilders.boolQuery(); + for (String monitorId : monitorIds) { + monitorIdMatchQuery.should(QueryBuilders.matchPhraseQuery(ThreatIntelAlert.MONITOR_ID_FIELD, monitorId)); + + } + queryBuilder.filter(monitorIdMatchQuery); + + BoolQueryBuilder idMatchQuery = QueryBuilders.boolQuery(); + for (String id : request.getAlertIds()) { + idMatchQuery.should(QueryBuilders.matchPhraseQuery("_id", id)); + + } + queryBuilder.filter(idMatchQuery); + + if (request.getState() == Alert.State.COMPLETED) { + queryBuilder.filter(QueryBuilders.matchPhraseQuery(ThreatIntelAlert.STATE_FIELD, Alert.State.ACKNOWLEDGED.toString())); + } else if (request.getState() == Alert.State.ACKNOWLEDGED) { + queryBuilder.filter(QueryBuilders.matchPhraseQuery(ThreatIntelAlert.STATE_FIELD, Alert.State.ACTIVE.toString())); + } else { + log.error("Threat intel monitor not found. No alerts to update"); + listener.onFailure(new SecurityAnalyticsException("Threat intel monitor not found. No alerts to update", + RestStatus.BAD_REQUEST, + new IllegalArgumentException("Threat intel monitor not found. No alerts to update"))); + return null; + } + + + return new SearchSourceBuilder() + .version(true) + .seqNoAndPrimaryTerm(true) + .query(queryBuilder) + .size(request.getAlertIds().size()); + } + + private void updateAlerts(List monitorIds, List alerts, Alert.State state, ActionListener listener) { + List failedAlerts = new ArrayList<>(); + List alertsToUpdate = new ArrayList<>(); + for (ThreatIntelAlert alert : alerts) { + if (isValidStateTransitionRequested(alert.getState(), state)) { + ThreatIntelAlert updatedAlertModel = ThreatIntelAlert.updateStatus(alert, state); + alertsToUpdate.add(updatedAlertModel); + } else { + log.error("Alert {} : updating alert state from {} to {} is not allowed!", alert.getId(), alert.getState(), state); + failedAlerts.add(alert.getId()); + } + } + alertsService.bulkIndexEntities(emptyList(), alertsToUpdate, ActionListener.wrap( + r -> { // todo change response to return failure messaages + List updatedAlerts = new ArrayList<>(); + SearchSourceBuilder searchSourceQueryingForAlerts = getSearchSourceQueryingForUpdatedAlerts( + monitorIds, + alertsToUpdate.stream().map(ThreatIntelAlert::getId).collect(Collectors.toList())); + alertsService.search(searchSourceQueryingForAlerts, ActionListener.wrap( + searchResponse -> { + if ( + searchResponse.getHits() == null || + searchResponse.getHits().getHits() == null || + searchResponse.getHits().getHits().length == 0 + ) { + log.error("No alerts found to update"); + listener.onFailure(new SecurityAnalyticsException("No alerts found to update", + RestStatus.BAD_REQUEST, + new ResourceNotFoundException("No alerts found to update"))); + return; + } + for (SearchHit hit : searchResponse.getHits().getHits()) { + XContentParser xcp = XContentType.JSON.xContent().createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString() + ); + if (xcp.currentToken() == null) + xcp.nextToken(); + if (xcp.currentToken() == null) + xcp.nextToken(); + ThreatIntelAlert alert = ThreatIntelAlert.parse(xcp, hit.getVersion()); + updatedAlerts.add(new ThreatIntelAlertDto(alert, hit.getSeqNo(), hit.getPrimaryTerm())); + } + listener.onResponse(new UpdateThreatIntelAlertsStatusResponse( + updatedAlerts, + failedAlerts + )); + }, + e -> { + log.error("Failed to fetch the updated alerts to return. Returning empty list for updated alerts although some might have been updated", e); + listener.onResponse(new UpdateThreatIntelAlertsStatusResponse( + emptyList(), + failedAlerts + )); + } + )); + + }, e -> { + log.error("Failed to bulk update status of threat intel alerts to " + state, e); + listener.onFailure(e); + } + )); + } + + private static SearchSourceBuilder getSearchSourceQueryingForUpdatedAlerts(List monitorIds, List alertIds) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + BoolQueryBuilder monitorIdMatchQuery = QueryBuilders.boolQuery(); + for (String monitorId : monitorIds) { + monitorIdMatchQuery.should(QueryBuilders.matchPhraseQuery(ThreatIntelAlert.MONITOR_ID_FIELD, monitorId)); + + } + queryBuilder.filter(monitorIdMatchQuery); + + BoolQueryBuilder idMatchQuery = QueryBuilders.boolQuery(); + for (String id : alertIds) { + idMatchQuery.should(QueryBuilders.matchPhraseQuery("_id", id)); + + } + queryBuilder.filter(idMatchQuery); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .version(true) + .seqNoAndPrimaryTerm(true) + .query(queryBuilder) + .size(alertIds.size()); + return searchSourceBuilder; + } + + private boolean isValidStateTransitionRequested(Alert.State currState, Alert.State nextState) { + if (currState.equals(Alert.State.ACKNOWLEDGED) && nextState.equals(Alert.State.COMPLETED)) { + return true; + } else if (currState.equals(Alert.State.ACTIVE) && nextState.equals(Alert.State.ACKNOWLEDGED)) { + return true; + } + return false; + } + + private void setFilterByEnabled(boolean filterByEnabled) { + this.filterByEnabled = filterByEnabled; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataUtils.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedDataUtils.java similarity index 96% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataUtils.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedDataUtils.java index a96558b50..20539695b 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataUtils.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedDataUtils.java @@ -2,7 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel; +package org.opensearch.securityanalytics.threatIntel.util; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedParser.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java similarity index 63% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedParser.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java index 92a66ed12..3cbf31086 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedParser.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java @@ -2,7 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel; +package org.opensearch.securityanalytics.threatIntel.util; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; @@ -12,7 +12,7 @@ import org.opensearch.SpecialPermission; import org.opensearch.common.SuppressForbidden; import org.opensearch.securityanalytics.threatIntel.common.Constants; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; import java.io.BufferedReader; import java.io.IOException; @@ -42,9 +42,27 @@ public static CSVParser getThreatIntelFeedReaderCSV(final TIFMetadata tifMetadat connection.addRequestProperty(Constants.USER_AGENT_KEY, Constants.USER_AGENT_VALUE); return new CSVParser(new BufferedReader(new InputStreamReader(connection.getInputStream())), CSVFormat.RFC4180); } catch (IOException e) { - log.error("Exception: failed to read threat intel feed data from {}",tifMetadata.getUrl(), e); + log.error("Exception: failed to read threat intel feed data from {}", tifMetadata.getUrl(), e); throw new OpenSearchException("failed to read threat intel feed data from {}", tifMetadata.getUrl(), e); } }); } + + /** + * Create CSVParser of a threat intel feed + */ + @SuppressForbidden(reason = "Need to connect to http endpoint to read threat intel feed database file") + public static CSVParser getThreatIntelFeedReaderCSV(URL url) { + SpecialPermission.check(); + return AccessController.doPrivileged((PrivilegedAction) () -> { + try { + URLConnection connection = url.openConnection(); + connection.addRequestProperty(Constants.USER_AGENT_KEY, Constants.USER_AGENT_VALUE); + return new CSVParser(new BufferedReader(new InputStreamReader(connection.getInputStream())), CSVFormat.RFC4180); + } catch (IOException e) { + log.error("Exception: failed to read threat intel feed data from {}", url, e); + throw new OpenSearchException("failed to read threat intel feed data from {}", url, e); + } + }); + } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelMonitorUtils.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelMonitorUtils.java new file mode 100644 index 000000000..912862940 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelMonitorUtils.java @@ -0,0 +1,201 @@ +package org.opensearch.securityanalytics.threatIntel.util; + +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.commons.alerting.model.Alert; +import org.opensearch.commons.alerting.model.Monitor; +import org.opensearch.commons.alerting.model.Trigger; +import org.opensearch.commons.alerting.model.remote.monitors.RemoteDocLevelMonitorInput; +import org.opensearch.commons.alerting.model.remote.monitors.RemoteMonitorTrigger; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; +import org.opensearch.securityanalytics.model.threatintel.ThreatIntelAlert; +import org.opensearch.securityanalytics.threatIntel.iocscan.dto.PerIocTypeScanInputDto; +import org.opensearch.securityanalytics.threatIntel.model.monitor.ThreatIntelInput; +import org.opensearch.securityanalytics.threatIntel.model.monitor.ThreatIntelTrigger; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelTriggerDto; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.opensearch.securityanalytics.util.XContentUtils.getBytesReference; + +public class ThreatIntelMonitorUtils { + public static RemoteMonitorTrigger buildRemoteMonitorTrigger(ThreatIntelTriggerDto trigger) throws IOException { + return new RemoteMonitorTrigger(trigger.getId(), trigger.getName(), trigger.getSeverity(), trigger.getActions(), + getBytesReference(new ThreatIntelTrigger(trigger.getDataSources(), trigger.getIocTypes()))); + } + + public static List buildThreatIntelTriggerDtos(List triggers, NamedXContentRegistry namedXContentRegistry) throws IOException { + + List triggerDtos = new ArrayList<>(); + for (Trigger trigger : triggers) { + RemoteMonitorTrigger remoteMonitorTrigger = (RemoteMonitorTrigger) trigger; + ThreatIntelTrigger threatIntelTrigger = getThreatIntelTriggerFromBytesReference(remoteMonitorTrigger, namedXContentRegistry); + + triggerDtos.add(new ThreatIntelTriggerDto( + threatIntelTrigger.getDataSources(), + threatIntelTrigger.getIocTypes(), + remoteMonitorTrigger.getActions(), + remoteMonitorTrigger.getName(), + remoteMonitorTrigger.getId(), + remoteMonitorTrigger.getSeverity() + )); + } + return triggerDtos; + } + + public static ThreatIntelTrigger getThreatIntelTriggerFromBytesReference(RemoteMonitorTrigger remoteMonitorTrigger, NamedXContentRegistry namedXContentRegistry) throws IOException { + StreamInput triggerSin = StreamInput.wrap(remoteMonitorTrigger.getTrigger().toBytesRef().bytes); + return new ThreatIntelTrigger(triggerSin); + } + + public static ThreatIntelInput getThreatIntelInputFromBytesReference(BytesReference bytes, NamedXContentRegistry namedXContentRegistry) throws IOException { + StreamInput sin = StreamInput.wrap(bytes.toBytesRef().bytes); + ThreatIntelInput threatIntelInput = new ThreatIntelInput(sin); + return threatIntelInput; + } + + public static ThreatIntelMonitorDto buildThreatIntelMonitorDto(String id, Monitor monitor, NamedXContentRegistry namedXContentRegistry) throws IOException { + RemoteDocLevelMonitorInput remoteDocLevelMonitorInput = (RemoteDocLevelMonitorInput) monitor.getInputs().get(0); + List indices = remoteDocLevelMonitorInput.getDocLevelMonitorInput().getIndices(); + ThreatIntelInput threatIntelInput = getThreatIntelInputFromBytesReference(remoteDocLevelMonitorInput.getInput(), namedXContentRegistry); + return new ThreatIntelMonitorDto( + id, + monitor.getName(), + threatIntelInput.getPerIocTypeScanInputList().stream().map(it -> new PerIocTypeScanInputDto(it.getIocType(), it.getIndexToFieldsMap())).collect(Collectors.toList()), + monitor.getSchedule(), + monitor.getEnabled(), + monitor.getUser(), + buildThreatIntelTriggerDtos(monitor.getTriggers(), namedXContentRegistry) + ); + } + + /** + * Fetch ACTIVE or ACKNOWLEDGED state alerts for the triggre. Criteria is they should match the ioc value+type from findings + */ + public static SearchSourceBuilder getSearchSourceBuilderForExistingAlertsQuery(ArrayList findings, Trigger trigger) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + queryBuilder.must(QueryBuilders.matchQuery(ThreatIntelAlert.TRIGGER_NAME_FIELD, trigger.getName())); + BoolQueryBuilder iocQueryBuilder = QueryBuilders.boolQuery(); + for (IocFinding finding : findings) { + BoolQueryBuilder innerQb = QueryBuilders.boolQuery(); + innerQb.must(QueryBuilders.matchQuery(ThreatIntelAlert.IOC_TYPE_FIELD, finding.getIocType())); + innerQb.must(QueryBuilders.matchQuery(ThreatIntelAlert.IOC_VALUE_FIELD, finding.getIocValue())); + iocQueryBuilder.should(innerQb); + } + queryBuilder.must(iocQueryBuilder); + BoolQueryBuilder stateQueryBuilder = QueryBuilders.boolQuery(); + stateQueryBuilder.should(QueryBuilders.matchQuery(ThreatIntelAlert.STATE_FIELD, Alert.State.ACTIVE.toString())); + stateQueryBuilder.should(QueryBuilders.matchQuery(ThreatIntelAlert.STATE_FIELD, Alert.State.ACKNOWLEDGED.toString())); + queryBuilder.must(stateQueryBuilder); + + SearchSourceBuilder ssb = new SearchSourceBuilder(); + ssb.query(queryBuilder); + ssb.size(9999); + return ssb; + } + + + public static Map prepareAlertsToUpdate(ArrayList triggerMatchedFindings, + List existingAlerts) { + Map updatedAlerts = new HashMap<>(); + for (ThreatIntelAlert existingAlert : existingAlerts) { + String iocType = existingAlert.getIocType(); + String iocValue = existingAlert.getIocValue(); + if (iocType == null || iocValue == null) + continue; + for (IocFinding finding : triggerMatchedFindings) { + if (iocType.equals(finding.getIocType()) && iocValue.equals(finding.getIocValue())) { + List findingIds = new ArrayList<>(existingAlert.getFindingIds()); + findingIds.add(finding.getId()); + updatedAlerts.put(existingAlert.getIocValue() + existingAlert.getIocType(), new ThreatIntelAlert(existingAlert, findingIds)); + } + } + } + return updatedAlerts; + + } + + public static List prepareNewAlerts(Monitor monitor, + Trigger trigger, + ArrayList findings, + Map updatedAlerts) { + List alerts = new ArrayList<>(); + for (IocFinding finding : findings) { + if (updatedAlerts.containsKey(finding.getIocValue() + finding.getIocType())) + continue; + Instant now = Instant.now(); + alerts.add(new ThreatIntelAlert( + UUID.randomUUID().toString(), + ThreatIntelAlert.NO_VERSION, + ThreatIntelAlert.NO_SCHEMA_VERSION, + monitor.getUser(), + trigger.getId(), + trigger.getName(), + monitor.getId(), + monitor.getName(), + Alert.State.ACTIVE, + now, + null, + now, + null, + null, + trigger.getSeverity(), + finding.getIocValue(), + finding.getIocType(), + Collections.emptyList(), + List.of(finding.getId()) + )); + } + return alerts; + } + + public static ArrayList getTriggerMatchedFindings(List iocFindings, ThreatIntelTrigger threatIntelTrigger) { + ArrayList triggerMatchedFindings = new ArrayList(); + for (IocFinding iocFinding : iocFindings) { + boolean iocTypeConditionMatch = false; + if (threatIntelTrigger.getIocTypes() == null || threatIntelTrigger.getIocTypes().isEmpty()) { + iocTypeConditionMatch = true; + } else if (threatIntelTrigger.getIocTypes().contains(iocFinding.getIocType().toLowerCase())) { + iocTypeConditionMatch = true; + } + boolean dataSourcesConditionMatch = false; + if (threatIntelTrigger.getDataSources() == null || threatIntelTrigger.getDataSources().isEmpty()) { + dataSourcesConditionMatch = true; + } else { + List dataSources = iocFinding.getRelatedDocIds().stream().map(it -> { + String[] parts = it.split(":"); + if (parts.length == 2) { + return parts[1]; + } else return null; + }).filter(Objects::nonNull).collect(Collectors.toList()); + if (threatIntelTrigger.getDataSources().stream().anyMatch(dataSources::contains)) { + dataSourcesConditionMatch = true; + } + } + if (dataSourcesConditionMatch && iocTypeConditionMatch) { + triggerMatchedFindings.add(iocFinding); + } + } + return triggerMatchedFindings; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportAckCorrelationAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportAckCorrelationAlertsAction.java index 917d0349c..7032819de 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportAckCorrelationAlertsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportAckCorrelationAlertsAction.java @@ -67,6 +67,8 @@ protected void doExecute(Task task, AckCorrelationAlertsRequest request, ActionL return; } + this.threadPool.getThreadContext().stashContext(); + if (!request.getCorrelationAlertIds().isEmpty()) { correlationAlertService.acknowledgeAlerts( request.getCorrelationAlertIds(), diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportAcknowledgeAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportAcknowledgeAlertsAction.java index 16679e9b2..0018a0f18 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportAcknowledgeAlertsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportAcknowledgeAlertsAction.java @@ -82,6 +82,8 @@ public void onResponse(GetDetectorResponse getDetectorResponse) { request.getAlertIds(), getDetectorResponse.getDetector(), new Table("asc", "id", null, 10000, 0, null), + null, + null, getAlertsResponseStepListener ); getAlertsResponseStepListener.whenComplete(getAlertsResponse -> { diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportGetAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportGetAlertsAction.java index f01929fc9..c2bdd7a15 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportGetAlertsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportGetAlertsAction.java @@ -91,6 +91,8 @@ protected void doExecute(Task task, GetAlertsRequest request, ActionListener docLevelQueries.add(docLevelQuery); } docLevelQueries.addAll(threatIntelQueries); - DocLevelMonitorInput docLevelMonitorInput = new DocLevelMonitorInput(detector.getName(), detector.getInputs().get(0).getIndices(), docLevelQueries); + DocLevelMonitorInput docLevelMonitorInput = new DocLevelMonitorInput(detector.getName(), detector.getInputs().get(0).getIndices(), docLevelQueries, true); docLevelMonitorInputs.add(docLevelMonitorInput); List triggers = new ArrayList<>(); @@ -784,7 +788,7 @@ private IndexMonitorRequest createDocLevelMonitorRequest(List } Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, detector.getName(), false, detector.getSchedule(), detector.getLastUpdateTime(), null, - Monitor.MonitorType.DOC_LEVEL_MONITOR, detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), + Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), new DataSources(detector.getRuleIndex(), detector.getFindingsIndex(), detector.getFindingsIndexPattern(), @@ -792,7 +796,7 @@ private IndexMonitorRequest createDocLevelMonitorRequest(List detector.getAlertsHistoryIndex(), detector.getAlertsHistoryIndexPattern(), DetectorMonitorConfig.getRuleIndexMappingsByType(), - true), PLUGIN_OWNER_FIELD); + true), enableDetectorWithDedicatedQueryIndices, null, PLUGIN_OWNER_FIELD); return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); } @@ -854,21 +858,26 @@ private IndexMonitorRequest createDocLevelMonitorMatchAllRequest( if(query.getRight().isAggregationRule()) { Rule rule = query.getRight(); tags.add(rule.getLevel()); + tags.add(rule.getId()); tags.add(rule.getCategory()); tags.addAll(rule.getTags().stream().map(Value::getValue).collect(Collectors.toList())); } } tags.removeIf(Objects::isNull); + + // if queryFieldNames is not passed, alerting doc-level monitor fetches entire log doc. + List queryFieldNames = List.of("_id"); DocLevelQuery docLevelQuery = new DocLevelQuery( monitorName, monitorName + "doc", Collections.emptyList(), actualQuery, - new ArrayList<>(tags) + new ArrayList<>(tags), + queryFieldNames ); docLevelQueries.add(docLevelQuery); - DocLevelMonitorInput docLevelMonitorInput = new DocLevelMonitorInput(detector.getName(), detector.getInputs().get(0).getIndices(), docLevelQueries); + DocLevelMonitorInput docLevelMonitorInput = new DocLevelMonitorInput(detector.getName(), detector.getInputs().get(0).getIndices(), docLevelQueries, false); docLevelMonitorInputs.add(docLevelMonitorInput); List triggers = new ArrayList<>(); @@ -879,21 +888,21 @@ private IndexMonitorRequest createDocLevelMonitorMatchAllRequest( String name = detectorTrigger.getName(); String severity = detectorTrigger.getSeverity(); List actions = detectorTrigger.getActions(); - Script condition = detectorTrigger.convertToCondition(); + Script condition = detectorTrigger.convertToConditionForChainedFindings(); triggers.add(new DocumentLevelTrigger(id, name, severity, actions, condition)); } Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, monitorName, false, detector.getSchedule(), detector.getLastUpdateTime(), null, - Monitor.MonitorType.DOC_LEVEL_MONITOR, detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), - new DataSources(detector.getRuleIndex(), + Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), + new DataSources(enableDetectorWithDedicatedQueryIndices? detector.getRuleIndex() + "_chained_findings": detector.getRuleIndex(), detector.getFindingsIndex(), detector.getFindingsIndexPattern(), detector.getAlertsIndex(), detector.getAlertsHistoryIndex(), detector.getAlertsHistoryIndexPattern(), DetectorMonitorConfig.getRuleIndexMappingsByType(), - true), PLUGIN_OWNER_FIELD); + true), enableDetectorWithDedicatedQueryIndices, true, PLUGIN_OWNER_FIELD); return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); } @@ -1033,6 +1042,8 @@ public void onResponse(GetIndexMappingsResponse getIndexMappingsResponse) { boolQueryBuilder.must(timeRangeFilter); searchSourceBuilder.query(boolQueryBuilder); } + // query hits are not needed from this query part for aggregations. + searchSourceBuilder.size(0); List bucketLevelMonitorInputs = new ArrayList<>(); bucketLevelMonitorInputs.add(new SearchInput(indices, searchSourceBuilder)); @@ -1055,7 +1066,7 @@ public void onResponse(GetIndexMappingsResponse getIndexMappingsResponse) { } **/ Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, detector.getName(), false, detector.getSchedule(), detector.getLastUpdateTime(), null, - MonitorType.BUCKET_LEVEL_MONITOR, detector.getUser(), 1, bucketLevelMonitorInputs, triggers, Map.of(), + MonitorType.BUCKET_LEVEL_MONITOR.getValue(), detector.getUser(), 1, bucketLevelMonitorInputs, triggers, Map.of(), new DataSources(detector.getRuleIndex(), detector.getFindingsIndex(), detector.getFindingsIndexPattern(), @@ -1063,7 +1074,7 @@ public void onResponse(GetIndexMappingsResponse getIndexMappingsResponse) { detector.getAlertsHistoryIndex(), detector.getAlertsHistoryIndexPattern(), DetectorMonitorConfig.getRuleIndexMappingsByType(), - true), PLUGIN_OWNER_FIELD); + true), false, null, PLUGIN_OWNER_FIELD); listener.onResponse(new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null)); } @@ -1247,7 +1258,13 @@ void createDetector() { request.getDetector().setAlertsHistoryIndexPattern(DetectorMonitorConfig.getAlertsHistoryIndexPattern(ruleTopic)); request.getDetector().setFindingsIndex(DetectorMonitorConfig.getFindingsIndex(ruleTopic)); request.getDetector().setFindingsIndexPattern(DetectorMonitorConfig.getFindingsIndexPattern(ruleTopic)); - request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); + + if (enableDetectorWithDedicatedQueryIndices) { + // disabling the setting after enabling it will mean delete & re-create the detector + request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndexOptimized(ruleTopic)); + } else { + request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); + } User originalContextUser = this.user; log.debug("user from original context is {}", originalContextUser); @@ -1364,7 +1381,16 @@ void onGetResponse(Detector currentDetector, User user) { request.getDetector().setAlertsHistoryIndexPattern(DetectorMonitorConfig.getAlertsHistoryIndexPattern(ruleTopic)); request.getDetector().setFindingsIndex(DetectorMonitorConfig.getFindingsIndex(ruleTopic)); request.getDetector().setFindingsIndexPattern(DetectorMonitorConfig.getFindingsIndexPattern(ruleTopic)); - request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); + if (currentDetector.getRuleIndex().contains("optimized")) { + request.getDetector().setRuleIndex(currentDetector.getRuleIndex()); + } else { + if (enableDetectorWithDedicatedQueryIndices) { + // disabling the setting after enabling it will mean delete & re-create the detector + request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndexOptimized(ruleTopic)); + } else { + request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); + } + } request.getDetector().setUser(user); if (!detector.getInputs().isEmpty()) { @@ -1777,7 +1803,7 @@ private Map mapMonitorIds(List monitorResp Collectors.toMap( // In the case of bucket level monitors rule id is trigger id it -> { - if (MonitorType.BUCKET_LEVEL_MONITOR == it.getMonitor().getMonitorType()) { + if (MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(it.getMonitor().getMonitorType())) { return it.getMonitor().getTriggers().get(0).getId(); } else { if (it.getMonitor().getName().contains("_chained_findings")) { @@ -1800,4 +1826,8 @@ private void setFilterByEnabled(boolean filterByEnabled) { private void setEnabledWorkflowUsage(boolean enabledWorkflowUsage) { this.enabledWorkflowUsage = enabledWorkflowUsage; } + + private void setEnabledDetectorsWithDedicatedQueryIndices(boolean enabledDetectorsWithDedicatedQueryIndices) { + this.enableDetectorWithDedicatedQueryIndices = enabledDetectorsWithDedicatedQueryIndices; + } } diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchDetectorAction.java index 3b7b36503..5937769fe 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchDetectorAction.java @@ -19,7 +19,7 @@ import org.opensearch.securityanalytics.action.SearchDetectorAction; import org.opensearch.securityanalytics.action.SearchDetectorRequest; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; -import org.opensearch.securityanalytics.threatIntel.action.TransportPutTIFJobAction; +import org.opensearch.securityanalytics.threatIntel.transport.TransportPutTIFJobAction; import org.opensearch.securityanalytics.util.DetectorIndices; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportTestS3ConnectionAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportTestS3ConnectionAction.java new file mode 100644 index 000000000..bd8f1c3a6 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportTestS3ConnectionAction.java @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.common.inject.Inject; +import org.opensearch.core.action.ActionListener; +import org.opensearch.securityanalytics.action.TestS3ConnectionAction; +import org.opensearch.securityanalytics.action.TestS3ConnectionRequest; +import org.opensearch.securityanalytics.action.TestS3ConnectionResponse; +import org.opensearch.securityanalytics.services.STIX2IOCFetchService; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import org.opensearch.tasks.Task; +import org.opensearch.transport.TransportService; + +public class TransportTestS3ConnectionAction extends HandledTransportAction implements SecureTransportAction { + + private static final Logger log = LogManager.getLogger(TransportTestS3ConnectionAction.class); + + private final STIX2IOCFetchService stix2IOCFetchService; + + @Inject + public TransportTestS3ConnectionAction( + TransportService transportService, + ActionFilters actionFilters, + STIX2IOCFetchService stix2IOCFetchService + ) { + super(TestS3ConnectionAction.NAME, transportService, actionFilters, TestS3ConnectionRequest::new); + this.stix2IOCFetchService = stix2IOCFetchService; + } + + @Override + protected void doExecute(Task task, TestS3ConnectionRequest request, ActionListener listener) { + try { + stix2IOCFetchService.testS3Connection(request.constructS3ConnectorConfig(), listener); + } catch (Exception e) { + log.warn("S3 connection test failed with error: ", e); + listener.onFailure(SecurityAnalyticsException.wrap(e)); + } + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/util/CorrelationIndices.java b/src/main/java/org/opensearch/securityanalytics/util/CorrelationIndices.java index 375342d09..36fd5e37d 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/CorrelationIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/CorrelationIndices.java @@ -7,6 +7,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.admin.indices.alias.Alias; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.core.action.ActionListener; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexResponse; @@ -26,6 +27,9 @@ import java.nio.charset.Charset; import java.util.Objects; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; + public class CorrelationIndices { private static final Logger log = LogManager.getLogger(CorrelationIndices.class); @@ -55,9 +59,15 @@ public static String correlationMappings() throws IOException { public void initCorrelationIndex(ActionListener actionListener) throws IOException { if (!correlationIndexExists()) { + Settings indexSettings = Settings.builder() + .put("index.hidden", true) + .put("index.correlation", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) + .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(CORRELATION_HISTORY_INDEX_PATTERN) .mapping(correlationMappings()) - .settings(Settings.builder().put("index.hidden", true).put("index.correlation", true).build()); + .settings(indexSettings); indexRequest.alias(new Alias(CORRELATION_HISTORY_WRITE_INDEX)); client.admin().indices().create(indexRequest, actionListener); } else { @@ -67,9 +77,15 @@ public void initCorrelationIndex(ActionListener actionListe public void initCorrelationMetadataIndex(ActionListener actionListener) throws IOException { if (!correlationMetadataIndexExists()) { + Settings indexSettings = Settings.builder() + .put("index.hidden", true) + .put("index.correlation", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) + .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(CORRELATION_METADATA_INDEX) .mapping(correlationMappings()) - .settings(Settings.builder().put("index.hidden", true).put("index.correlation", true).build()); + .settings(indexSettings); client.admin().indices().create(indexRequest, actionListener); } else { actionListener.onResponse(new CreateIndexResponse(true, true, CORRELATION_METADATA_INDEX)); @@ -136,6 +152,8 @@ public static String correlationAlertIndexMappings() throws IOException { public void initCorrelationAlertIndex(ActionListener actionListener) throws IOException { Settings correlationAlertSettings = Settings.builder() .put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(CORRELATION_ALERT_INDEX) .mapping(correlationAlertIndexMappings()) diff --git a/src/main/java/org/opensearch/securityanalytics/util/CorrelationRuleIndices.java b/src/main/java/org/opensearch/securityanalytics/util/CorrelationRuleIndices.java index d131e47b4..27f6475f8 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/CorrelationRuleIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/CorrelationRuleIndices.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.core.action.ActionListener; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexResponse; @@ -23,6 +24,9 @@ import java.util.Objects; import org.opensearch.securityanalytics.model.CorrelationRule; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; + public class CorrelationRuleIndices { private static final Logger log = LogManager.getLogger(CorrelationRuleIndices.class); @@ -45,9 +49,14 @@ public static String correlationRuleIndexMappings() throws IOException { public void initCorrelationRuleIndex(ActionListener actionListener) throws IOException { if (!correlationRuleIndexExists()) { + Settings indexSettings = Settings.builder() + .put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) + .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(CorrelationRule.CORRELATION_RULE_INDEX).mapping( correlationRuleIndexMappings() - ).settings(Settings.builder().put("index.hidden", true).build()); + ).settings(indexSettings); client.admin().indices().create(indexRequest, actionListener); } } diff --git a/src/main/java/org/opensearch/securityanalytics/util/CustomLogTypeIndices.java b/src/main/java/org/opensearch/securityanalytics/util/CustomLogTypeIndices.java index 21eb460d5..d2b5661fe 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/CustomLogTypeIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/CustomLogTypeIndices.java @@ -21,6 +21,8 @@ import java.io.IOException; import java.nio.charset.Charset; import java.util.Objects; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; public class CustomLogTypeIndices { @@ -42,9 +44,11 @@ public static String customLogTypeMappings() throws IOException { public void initCustomLogTypeIndex(ActionListener actionListener) throws IOException { if (!customLogTypeIndexExists()) { + // Security Analytics log types index is small. 1 primary shard is enough Settings indexSettings = Settings.builder() .put("index.hidden", true) - .put("index.auto_expand_replicas", "0-all") + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(LogTypeService.LOG_TYPE_INDEX) .mapping(customLogTypeMappings()) diff --git a/src/main/java/org/opensearch/securityanalytics/util/DetectorIndices.java b/src/main/java/org/opensearch/securityanalytics/util/DetectorIndices.java index d6a81e134..83eb058e0 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/DetectorIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/DetectorIndices.java @@ -23,6 +23,9 @@ import java.nio.charset.Charset; import java.util.Objects; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; + public class DetectorIndices { private static final Logger log = LogManager.getLogger(DetectorIndices.class); @@ -45,9 +48,14 @@ public static String detectorMappings() throws IOException { public void initDetectorIndex(ActionListener actionListener) throws IOException { if (!detectorIndexExists()) { + Settings indexSettings = Settings.builder() + .put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) + .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(Detector.DETECTORS_INDEX) .mapping(detectorMappings()) - .settings(Settings.builder().put("index.hidden", true).build()); + .settings(indexSettings); client.indices().create(indexRequest, actionListener); } } diff --git a/src/main/java/org/opensearch/securityanalytics/util/DetectorUtils.java b/src/main/java/org/opensearch/securityanalytics/util/DetectorUtils.java index 14c241f83..2a3bb041a 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/DetectorUtils.java +++ b/src/main/java/org/opensearch/securityanalytics/util/DetectorUtils.java @@ -109,7 +109,7 @@ public static List getBucketLevelMonitorIds( ) { return monitorResponses.stream().filter( // In the case of bucket level monitors rule id is trigger id - it -> Monitor.MonitorType.BUCKET_LEVEL_MONITOR == it.getMonitor().getMonitorType() + it -> Monitor.MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(it.getMonitor().getMonitorType()) ).map(IndexMonitorResponse::getId).collect(Collectors.toList()); } public static List getAggRuleIdsConfiguredToTrigger(Detector detector, List> rulesById) { diff --git a/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java b/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java index a24286fda..c4dafed85 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java +++ b/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java @@ -4,11 +4,6 @@ */ package org.opensearch.securityanalytics.util; -import java.util.Optional; -import java.util.SortedMap; - -import org.opensearch.cluster.metadata.Metadata; -import org.opensearch.core.action.ActionListener; import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.master.AcknowledgedResponse; @@ -17,23 +12,28 @@ import org.opensearch.cluster.metadata.IndexAbstraction; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.action.ActionListener; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; -import java.util.function.Predicate; +import java.util.Optional; +import java.util.SortedMap; public class IndexUtils { - private static final String _META = "_meta"; - private static final Integer NO_SCHEMA_VERSION = 0; - private static final String SCHEMA_VERSION = "schema_version"; + public static final String _META = "_meta"; + public static final Integer NO_SCHEMA_VERSION = 0; + public static final String SCHEMA_VERSION = "schema_version"; public static Boolean detectorIndexUpdated = false; public static Boolean customRuleIndexUpdated = false; @@ -210,4 +210,25 @@ public static String getIndexNameWithAlias(ClusterState clusterState, String ali return entry.map(Map.Entry::getKey).orElse(null); } + public static Map> getConcreteindexToMonitorInputIndicesMap(List indices, ClusterService clusterService, IndexNameExpressionResolver resolver) { + Map> result = new HashMap<>(); + + for (String index : indices) { + String[] concreteIndices = resolver.concreteIndexNames( + clusterService.state(), + IndicesOptions.lenientExpand(), + true, + index + ); + for (String concreteIndex : concreteIndices) { + if (!result.containsKey(concreteIndex)) { + result.put(concreteIndex, new ArrayList<>()); + } + result.get(concreteIndex).add(index); + } + } + + return result; + } + } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java index 94a98106b..b1ff516d5 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java @@ -65,6 +65,8 @@ import java.util.stream.Stream; import static org.opensearch.securityanalytics.model.Detector.NO_VERSION; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; public class RuleIndices { @@ -93,6 +95,8 @@ public void initRuleIndex(ActionListener actionListener, bo if (!ruleIndexExists(isPrepackaged)) { Settings indexSettings = Settings.builder() .put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(getRuleIndex(isPrepackaged)) .mapping(ruleMappings()) diff --git a/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java b/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java index 8f0f1cab5..7aa2def36 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java @@ -46,13 +46,9 @@ public static String ruleTopicIndexSettings() throws IOException { public void initRuleTopicIndexTemplate(ActionListener actionListener) throws IOException { getAllRuleIndices(ActionListener.wrap(allRuleIndices -> { // Compose list of all patterns to cover all query indices - List indexPatterns = new ArrayList<>(); - for(String ruleIndex : allRuleIndices) { - indexPatterns.add(ruleIndex + "*"); - } ComposableIndexTemplate template = new ComposableIndexTemplate( - indexPatterns, + allRuleIndices, new Template( Settings.builder().loadFromSource(ruleTopicIndexSettings(), XContentType.JSON).build(), null, @@ -87,7 +83,8 @@ private void getAllRuleIndices(ActionListener> listener) { listener.onResponse( logTypes .stream() - .map(logType -> DetectorMonitorConfig.getRuleIndex(logType)) + // use index pattern here to define rule topic index template for all query indices which match the pattern + .map(logType -> DetectorMonitorConfig.getRuleIndex(logType) + "*") .collect(Collectors.toList()) ); }, listener::onFailure)); diff --git a/src/main/java/org/opensearch/securityanalytics/util/WorkflowService.java b/src/main/java/org/opensearch/securityanalytics/util/WorkflowService.java index fa19d9958..aa5d547c8 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/WorkflowService.java +++ b/src/main/java/org/opensearch/securityanalytics/util/WorkflowService.java @@ -100,6 +100,11 @@ public void upsertWorkflow( } cmfMonitorId = addedMonitorResponses.stream().filter(res -> (detector.getName() + "_chained_findings").equals(res.getMonitor().getName())).findFirst().get().getId(); chainedMonitorFindings = new ChainedMonitorFindings(null, getBucketLevelMonitorIds(monitorResponses)); + } else if (updatedMonitorResponses != null && updatedMonitorResponses.stream().anyMatch(res -> (detector.getName() + "_chained_findings").equals(res.getMonitor().getName()))) { + List monitorResponses = new ArrayList<>(updatedMonitorResponses); + monitorResponses.addAll(updatedMonitorResponses); + cmfMonitorId = updatedMonitorResponses.stream().filter(res -> (detector.getName() + "_chained_findings").equals(res.getMonitor().getName())).findFirst().get().getId(); + chainedMonitorFindings = new ChainedMonitorFindings(null, getBucketLevelMonitorIds(monitorResponses)); } IndexWorkflowRequest indexWorkflowRequest = createWorkflowRequest(monitorIds, diff --git a/src/main/java/org/opensearch/securityanalytics/util/XContentUtils.java b/src/main/java/org/opensearch/securityanalytics/util/XContentUtils.java index 5389758af..6c56af6fc 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/XContentUtils.java +++ b/src/main/java/org/opensearch/securityanalytics/util/XContentUtils.java @@ -6,13 +6,18 @@ package org.opensearch.securityanalytics.util; import java.io.IOException; +import java.time.Instant; +import java.util.Locale; import java.util.Map; + +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; -import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; public class XContentUtils { @@ -27,4 +32,32 @@ public static String parseMapToJsonString(Map map) throws IOExce ); } + public static BytesReference getBytesReference(Writeable writeable) throws IOException { + BytesStreamOutput out = new BytesStreamOutput(); + writeable.writeTo(out); + BytesReference bytes = out.bytes(); + return bytes; + } + + public static Instant getInstant(XContentParser xcp) throws IOException { + Instant lastUpdateTime; + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + lastUpdateTime = null; + } else if (xcp.currentToken().isValue()) { + lastUpdateTime = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + lastUpdateTime = null; + } + return lastUpdateTime; + } + + public static void buildInstantAsField(XContentBuilder builder, Instant instant, String fieldName) throws IOException { + if (instant == null) { + builder.nullField(fieldName); + } else { + builder.timeField(fieldName, String.format(Locale.getDefault(), "%s_in_millis", fieldName), instant.toEpochMilli()); + } + } + } \ No newline at end of file diff --git a/src/main/plugin-metadata/plugin-security.policy b/src/main/plugin-metadata/plugin-security.policy index bcee5e9e6..3133ecd3b 100644 --- a/src/main/plugin-metadata/plugin-security.policy +++ b/src/main/plugin-metadata/plugin-security.policy @@ -1,3 +1,11 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + grant { // needed to find the classloader to load whitelisted classes. permission java.lang.RuntimePermission "createClassLoader"; @@ -5,4 +13,11 @@ grant { permission java.net.SocketPermission "*", "connect,resolve"; permission java.net.NetPermission "getProxySelector"; + + // Needed to make calls to AWS S3 + permission java.io.FilePermission "${user.home}${/}.aws${/}*", "read"; + + // Needed to parse response from AWS S3 + permission java.lang.RuntimePermission "accessDeclaredMembers"; + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; }; diff --git a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index e4a14ac56..f196c4a5a 100644 --- a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -1,2 +1,3 @@ org.opensearch.securityanalytics.correlation.index.codec.correlation950.CorrelationCodec950 -org.opensearch.securityanalytics.correlation.index.codec.correlation990.CorrelationCodec990 \ No newline at end of file +org.opensearch.securityanalytics.correlation.index.codec.correlation990.CorrelationCodec990 +org.opensearch.securityanalytics.correlation.index.codec.correlation9120.CorrelationCodec9120 \ No newline at end of file diff --git a/src/main/resources/META-INF/services/org.opensearch.alerting.spi.RemoteMonitorRunnerExtension b/src/main/resources/META-INF/services/org.opensearch.alerting.spi.RemoteMonitorRunnerExtension new file mode 100644 index 000000000..288e984da --- /dev/null +++ b/src/main/resources/META-INF/services/org.opensearch.alerting.spi.RemoteMonitorRunnerExtension @@ -0,0 +1,6 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# + +org.opensearch.securityanalytics.SecurityAnalyticsPlugin \ No newline at end of file diff --git a/src/main/resources/OSMapping/cloudtrail_logtype.json b/src/main/resources/OSMapping/cloudtrail_logtype.json index 8c2ea3b3a..cb9c80440 100644 --- a/src/main/resources/OSMapping/cloudtrail_logtype.json +++ b/src/main/resources/OSMapping/cloudtrail_logtype.json @@ -34,7 +34,8 @@ { "raw_field":"eventType", "ecs":"aws.cloudtrail.event_type", - "ocsf": "unmapped.eventType" + "ocsf" : "unmapped.eventType", + "ocsf11": "metadata.event_code" }, { "raw_field":"eventCategory", @@ -69,7 +70,8 @@ { "raw_field":"additionalEventData.MFAUsed", "ecs":"aws.cloudtrail.additional_event_data.mfaUsed", - "ocsf": "mfa" + "ocsf": "mfa", + "ocsf11": "is_mfa" }, { "raw_field":"responseElements", @@ -124,12 +126,14 @@ { "raw_field":"requestParameters.userName", "ecs":"aws.cloudtrail.request_parameters.username", - "ocsf": "unmapped.requestParameters.userName" + "ocsf": "unmapped.requestParameters.userName", + "ocsf11": "user.name" }, { "raw_field":"requestParameters.roleArn", "ecs":"aws.cloudtrail.request_parameters.roleArn", - "ocsf": "user.uuid" + "ocsf": "user.uuid", + "ocsf11": "user.uid" }, { "raw_field":"requestParameters.roleSessionName", @@ -149,17 +153,20 @@ { "raw_field":"userIdentity.principalId", "ecs":"aws.cloudtrail.user_identity.principalId", - "ocsf": "actor.user.uid" + "ocsf": "actor.user.uid", + "ocsf11":"actor.user.uid_alt" }, { "raw_field":"userIdentity.arn", "ecs":"aws.cloudtrail.user_identity.arn", - "ocsf": "actor.user.uuid" + "ocsf": "actor.user.uuid", + "ocsf11": "actor.user.uid" }, { "raw_field":"userIdentity.accountId", "ecs":"aws.cloudtrail.user_identity.accountId", - "ocsf": "actor.user.account_uid" + "ocsf": "actor.user.account_uid", + "ocsf11": "actor.user.account.uid" }, { "raw_field":"userIdentity.accessKeyId", @@ -199,7 +206,8 @@ { "raw_field":"userIdentity.sessionContext.attributes.mfaAuthenticated", "ecs":"aws.cloudtrail.user_identity.session_context.attributes.mfaAuthenticated", - "ocsf": "actor.session.mfa" + "ocsf": "actor.session.mfa", + "ocsf11": "actor.session.is_mfa" }, { "raw_field":"userIdentity.webIdFederationData.federatedProvider", diff --git a/src/main/resources/OSMapping/dns_logtype.json b/src/main/resources/OSMapping/dns_logtype.json index ef012407f..292002205 100644 --- a/src/main/resources/OSMapping/dns_logtype.json +++ b/src/main/resources/OSMapping/dns_logtype.json @@ -54,7 +54,8 @@ { "raw_field":"account_id", "ecs":"aws.route53.account_id", - "ocsf": "cloud.account_uid" + "ocsf": "cloud.account_uid", + "ocsf11": "cloud.account.uid" }, { "raw_field":"region", @@ -114,12 +115,23 @@ { "raw_field":"firewall_rule_action", "ecs":"aws.route53.srcids.firewall_rule_action", - "ocsf": "disposition_id" + "ocsf": "disposition_id", + "ocsf11": "disposition" }, { "raw_field":"creationTime", "ecs":"timestamp", "ocsf": "unmapped.creationTime" + }, + { + "raw_field":"rcode", + "ecs":"aws.route53.rcode", + "ocsf":"rcode" + }, + { + "raw_field":"firewall_rule_group_id", + "ecs":"aws.route53.srcids.firewall_rule_group_id", + "ocsf":"firewall_rule.uid" } ] } diff --git a/src/main/resources/OSMapping/network_logtype.json b/src/main/resources/OSMapping/network_logtype.json index 2ca92a1ad..2a247840b 100644 --- a/src/main/resources/OSMapping/network_logtype.json +++ b/src/main/resources/OSMapping/network_logtype.json @@ -14,131 +14,168 @@ "mappings":[ { "raw_field":"action", - "ecs":"netflow.firewall_event" + "ecs":"netflow.firewall_event", + "ocsf": "unmapped.action" }, { "raw_field":"certificate.serial", - "ecs":"zeek.x509.certificate.serial" + "ecs":"zeek.x509.certificate.serial", + "ocsf": "unmapped.certificate.serial" }, { "raw_field":"name", - "ecs":"zeek.smb_files.name" + "ecs":"zeek.smb_files.name", + "ocsf": "unmapped.name" }, { "raw_field":"path", - "ecs":"zeek.smb_files.path" + "ecs":"zeek.smb_files.path", + "ocsf": "unmapped.path" }, { "raw_field":"dst_port", - "ecs":"destination.port" + "ecs":"destination.port", + "ocsf": "unmapped.dst_port" }, { "raw_field":"qtype_name", - "ecs":"zeek.dns.qtype_name" + "ecs":"zeek.dns.qtype_name", + "ocsf": "query.type" }, { "raw_field":"operation", - "ecs":"zeek.dce_rpc.operation" + "ecs":"zeek.dce_rpc.operation", + "ocsf": "unmapped.operation" }, { "raw_field":"endpoint", - "ecs":"zeek.dce_rpc.endpoint" + "ecs":"zeek.dce_rpc.endpoint", + "ocsf": "unmapped.endpoint" }, { "raw_field":"zeek.dce_rpc.endpoint", - "ecs":"zeek.dce_rpc.endpoint" + "ecs":"zeek.dce_rpc.endpoint", + "ocsf": "unmapped.zeek.dce_rpc.endpoint" }, { "raw_field":"answers", - "ecs":"zeek.dns.answers" + "ecs":"zeek.dns.answers", + "ocsf": "answers.rdata" }, { "raw_field":"query", - "ecs":"zeek.dns.query" + "ecs":"zeek.dns.query", + "ocsf": "query.hostname" }, { "raw_field":"client_header_names", - "ecs":"zeek.http.client_header_names" + "ecs":"zeek.http.client_header_names", + "ocsf": "unmapped.client_header_names" }, { "raw_field":"resp_mime_types", - "ecs":"zeek.http.resp_mime_types" + "ecs":"zeek.http.resp_mime_types", + "ocsf": "unmapped.resp_mime_types" }, { "raw_field":"cipher", - "ecs":"zeek.kerberos.cipher" + "ecs":"zeek.kerberos.cipher", + "ocsf": "cipher" }, { "raw_field":"request_type", - "ecs":"zeek.kerberos.request_type" + "ecs":"zeek.kerberos.request_type", + "ocsf": "unmapped.request_type" }, { "raw_field":"creationTime", - "ecs":"timestamp" + "ecs":"timestamp", + "ocsf": "unmapped.creationTime" }, { "raw_field":"method", - "ecs":"http.request.method" + "ecs":"http.request.method", + "ocsf": "unmapped.method" }, { "raw_field":"id.resp_p", - "ecs":"id.resp_p" + "ecs":"id.resp_p", + "ocsf": "dst_endpoint.port" }, { "raw_field":"blocked", - "ecs":"blocked-flag" + "ecs":"blocked-flag", + "ocsf": "unmapped.blocked" + }, + { + "raw_field": "id.orig_p", + "ecs": "id.orig_p", + "ocsf": "src_endpoint.port" }, { "raw_field":"id.orig_h", - "ecs":"id.orig_h" + "ecs":"id.orig_h", + "ocsf": "src_endpoint.ip" }, { "raw_field":"Z", - "ecs":"Z-flag" + "ecs":"Z-flag", + "ocsf": "answers.flag_ids.99" }, { "raw_field":"id.resp_h", - "ecs":"id.resp_h" + "ecs":"id.resp_h", + "ocsf": "dst_endpoint.ip" }, { "raw_field":"uri", - "ecs":"url.path" + "ecs":"url.path", + "ocsf": "unmapped.uri" }, { "raw_field":"c-uri", - "ecs":"url.path" + "ecs":"url.path", + "ocsf": "unmapped.c-uri" }, { "raw_field":"c-useragent", - "ecs":"user_agent.name" + "ecs":"user_agent.name", + "ocsf": "unmapped.c-useragent" }, { "raw_field":"status_code", - "ecs":"http.response.status_code" + "ecs":"http.response.status_code", + "ocsf": "unmapped.status_code" }, { "raw_field":"rejected", - "ecs":"rejected" + "ecs":"rejected", + "ocsf": "unmapped.rejected" }, { "raw_field":"dst_ip", - "ecs":"destination.ip" + "ecs":"destination.ip", + "ocsf": "unmapped.dst_ip" }, { "raw_field":"src_ip", - "ecs":"source.ip" + "ecs":"source.ip", + "ocsf": "unmapped.src_ip" }, { "raw_field":"user_agent", - "ecs":"user_agent.name" + "ecs":"user_agent.name", + "ocsf": "unmapped.user_agent" }, { "raw_field":"request_body_len", - "ecs":"http.request.body.bytes" + "ecs":"http.request.body.bytes", + "ocsf": "unmapped.request_body_len" }, { "raw_field":"service", - "ecs":"service" + "ecs":"service", + "ocsf": "unmapped.service" } ] } diff --git a/src/main/resources/OSMapping/vpcflow_logtype.json b/src/main/resources/OSMapping/vpcflow_logtype.json index 29d9f38c2..762c2e5e7 100644 --- a/src/main/resources/OSMapping/vpcflow_logtype.json +++ b/src/main/resources/OSMapping/vpcflow_logtype.json @@ -20,7 +20,8 @@ { "raw_field":"account_id", "ecs":"netflow.account_id", - "ocsf": "cloud.account_uid" + "ocsf": "cloud.account_uid", + "ocsf11": "cloud.account.uid" }, { "raw_field":"region", @@ -90,12 +91,14 @@ { "raw_field":"action", "ecs":"netflow.action", - "ocsf": "disposition_id" + "ocsf": "disposition_id", + "ocsf11": "disposition" }, { "raw_field":"traffic_path", "ecs":"netflow.traffic_path", - "ocsf": "boundary_id" + "ocsf": "boundary_id", + "ocsf11": "connection_info.boundary_id" }, { "raw_field":"flow_direction", diff --git a/src/main/resources/OSMapping/waf_logtype.json b/src/main/resources/OSMapping/waf_logtype.json index 3e5b1f4f1..352a5e155 100644 --- a/src/main/resources/OSMapping/waf_logtype.json +++ b/src/main/resources/OSMapping/waf_logtype.json @@ -6,51 +6,63 @@ "mappings":[ { "raw_field":"cs-method", - "ecs":"waf.request.method" + "ecs":"waf.request.method", + "ocsf": "unmapped.cs-method" }, { "raw_field":"httpRequest.httpMethod", - "ecs":"waf.request.method" + "ecs":"waf.request.method", + "ocsf": "http_request.http_method" }, { "raw_field":"cs-uri-query", - "ecs":"waf.request.uri_query" + "ecs":"waf.request.uri_query", + "ocsf": "unmapped.cs-uri-query" }, { "raw_field":"httpRequest.uri", - "ecs":"waf.request.uri_query" + "ecs":"waf.request.uri_query", + "ocsf": "http_request.url.path" }, { "raw_field":"httpRequest.args", - "ecs":"waf.request.uri_query" + "ecs":"waf.request.uri_query", + "ocsf": "http_request.args" }, { "raw_field":"cs-user-agent", - "ecs":"waf.request.headers.user_agent" + "ecs":"waf.request.headers.user_agent", + "ocsf": "unmapped.cs-user-agent" }, { "raw_field":"httpRequest.headers", - "ecs":"waf.request.headers" + "ecs":"waf.request.headers", + "ocsf": "unmapped.httpRequest.headers" }, { "raw_field":"sc-status", - "ecs":"waf.response.code" + "ecs":"waf.response.code", + "ocsf": "unmapped.sc-status" }, { "raw_field":"responseCodeSent", - "ecs":"waf.response.code" + "ecs":"waf.response.code", + "ocsf": "status_code" }, { "raw_field":"timestamp", - "ecs":"timestamp" + "ecs":"timestamp", + "ocsf": "unmapped.timestamp" }, { "raw_field":"httpRequest.headers.value", - "ecs":"waf.request.headers.value" + "ecs":"waf.request.headers.value", + "ocsf": "http_request.http_headers.value" }, { "raw_field":"httpRequest.headers.name", - "ecs":"waf.request.headers.name" + "ecs":"waf.request.headers.name", + "ocsf": "http_request.http_headers.name" } ] } diff --git a/src/main/resources/OSMapping/windows_logtype.json b/src/main/resources/OSMapping/windows_logtype.json index ec9b3ed1a..1f68cc160 100644 --- a/src/main/resources/OSMapping/windows_logtype.json +++ b/src/main/resources/OSMapping/windows_logtype.json @@ -15,7 +15,8 @@ }, { "raw_field":"AuthenticationPackageName", - "ecs":"winlog.event_data.AuthenticationPackageName" + "ecs":"winlog.event_data.AuthenticationPackageName", + "ocsf": "auth_protocol" }, { "raw_field":"Channel", @@ -27,7 +28,8 @@ }, { "raw_field":"ComputerName", - "ecs":"winlog.computer_name" + "ecs":"winlog.computer_name", + "ocsf": "device.name" }, { "raw_field":"Description", @@ -71,11 +73,13 @@ }, { "raw_field":"LogonProcessName", - "ecs":"winlog.event_data.LogonProcessName" + "ecs":"winlog.event_data.LogonProcessName", + "ocsf": "logon_process.name" }, { "raw_field":"LogonType", - "ecs":"winlog.event_data.LogonType" + "ecs":"winlog.event_data.LogonType", + "ocsf": "logon_type_id" }, { "raw_field":"OriginalFilename", @@ -91,7 +95,8 @@ }, { "raw_field":"ProcessId", - "ecs":"winlog.event_data.ProcessId" + "ecs":"winlog.event_data.ProcessId", + "ocsf": "actor.process.pid" }, { "raw_field":"Product", @@ -127,11 +132,13 @@ }, { "raw_field":"Status", - "ecs":"winlog.event_data.Status" + "ecs":"winlog.event_data.Status", + "ocsf": "status" }, { "raw_field":"SubjectDomainName", - "ecs":"winlog.event_data.SubjectDomainName" + "ecs":"winlog.event_data.SubjectDomainName", + "ocsf": "actor.user.domain" }, { "raw_field":"SubjectLogonId", @@ -139,11 +146,13 @@ }, { "raw_field":"SubjectUserName", - "ecs":"winlog.event_data.SubjectUserName" + "ecs":"winlog.event_data.SubjectUserName", + "ocsf": "actor.user.name" }, { "raw_field":"SubjectUserSid", - "ecs":"winlog.event_data.SubjectUserSid" + "ecs":"winlog.event_data.SubjectUserSid", + "ocsf": "actor.user.uid" }, { "raw_field":"TargetLogonId", @@ -159,11 +168,13 @@ }, { "raw_field":"TargetUserName", - "ecs":"winlog.event_data.TargetUserName" + "ecs":"winlog.event_data.TargetUserName", + "ocsf": "process.user.domain" }, { "raw_field":"TargetUserSid", - "ecs":"winlog.event_data.TargetUserSid" + "ecs":"winlog.event_data.TargetUserSid", + "ocsf": "process.user.uid" }, { "raw_field":"TaskName", @@ -183,11 +194,13 @@ }, { "raw_field":"Workstation", - "ecs":"winlog.event_data.Workstation" + "ecs":"winlog.event_data.Workstation", + "ocsf": "src_endpoint.name" }, { "raw_field":"WorkstationName", - "ecs":"winlog.event_data.Workstation" + "ecs":"winlog.event_data.Workstation", + "ocsf": "src_endpoint.name" }, { "raw_field":"event_uid", @@ -219,7 +232,8 @@ }, { "raw_field":"ProcessName", - "ecs":"winlog.event_data.ProcessName" + "ecs":"winlog.event_data.ProcessName", + "ocsf": "actor.process.file" }, { "raw_field":"ObjectName", @@ -615,7 +629,8 @@ }, { "raw_field":"Message", - "ecs":"winlog.event_data.Message" + "ecs":"winlog.event_data.Message", + "ocsf": "message" }, { "raw_field":"ShareName", @@ -623,11 +638,13 @@ }, { "raw_field":"SourcePort", - "ecs":"source.port" + "ecs":"source.port", + "ocsf":"src_endpoint.port" }, { "raw_field":"CallerProcessName", - "ecs":"winlog.event_data.CallerProcessName" + "ecs":"winlog.event_data.CallerProcessName", + "ocsf": "actor.process.file" }, { "raw_field":"ServiceFileName", diff --git a/src/main/resources/mappings/ioc_finding_mapping.json b/src/main/resources/mappings/ioc_finding_mapping.json new file mode 100644 index 000000000..9a7deb67e --- /dev/null +++ b/src/main/resources/mappings/ioc_finding_mapping.json @@ -0,0 +1,52 @@ +{ + "dynamic": "strict", + "_meta" : { + "schema_version": 1 + }, + "properties": { + "schema_version": { + "type": "integer" + }, + "ioc_feed_ids" : { + "type": "object", + "properties": { + "feed_id": { + "type": "keyword" + }, + "feed_name": { + "type": "keyword" + }, + "ioc_id": { + "type": "keyword" + }, + "index": { + "type": "keyword" + } + } + }, + "related_doc_ids": { + "type": "keyword" + }, + "monitor_id": { + "type": "keyword" + }, + "monitor_name": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "ioc_value" : { + "type": "keyword" + }, + "ioc_type" : { + "type": "keyword" + }, + "timestamp": { + "type": "long" + }, + "execution_id": { + "type": "keyword" + } + } +} diff --git a/src/main/resources/mappings/stix2_ioc_mapping.json b/src/main/resources/mappings/stix2_ioc_mapping.json new file mode 100644 index 000000000..deaf7c4a1 --- /dev/null +++ b/src/main/resources/mappings/stix2_ioc_mapping.json @@ -0,0 +1,40 @@ +{ + "_meta": { + "schema_version": 1 + }, + "properties": { + "name": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "value": { + "type": "keyword" + }, + "severity": { + "type": "keyword" + }, + "spec_version": { + "type": "keyword" + }, + "created": { + "type": "date" + }, + "modified": { + "type": "date" + }, + "description": { + "type": "text" + }, + "labels": { + "type": "keyword" + }, + "feed_id": { + "type": "keyword" + }, + "feed_name": { + "type": "keyword" + } + } +} diff --git a/src/main/resources/mappings/threat_intel_alert_mapping.json b/src/main/resources/mappings/threat_intel_alert_mapping.json new file mode 100644 index 000000000..cfb030912 --- /dev/null +++ b/src/main/resources/mappings/threat_intel_alert_mapping.json @@ -0,0 +1,110 @@ +{ + "dynamic": "strict", + "_meta": { + "schema_version": 0 + }, + "properties": { + "id": { + "type": "keyword" + }, + "version": { + "type": "long" + }, + "schema_version": { + "type": "long" + }, + "user": { + "properties": { + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "backend_roles": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "roles": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "custom_attribute_names": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + } + } + }, + "trigger_id": { + "type": "keyword" + }, + "trigger_name": { + "type": "keyword" + }, + "monitor_id": { + "type": "keyword" + }, + "monitor_name": { + "type": "keyword" + }, + "state": { + "type": "keyword" + }, + "start_time": { + "type": "date" + }, + "end_time": { + "type": "date" + }, + "acknowledged_time": { + "type": "date" + }, + "last_updated_time": { + "type": "date" + }, + "error_message": { + "type": "text" + }, + "severity": { + "type": "keyword" + }, + "action_execution_results": { + "type": "nested", + "properties": { + "action_id": { + "type": "keyword" + }, + "last_execution_time": { + "type": "date" + }, + "throttled_count": { + "type": "integer" + } + } + }, + "ioc_value": { + "type": "keyword" + }, + "ioc_type": { + "type": "keyword" + }, + "finding_ids": { + "type": "text" + } + } +} diff --git a/src/main/resources/mappings/threat_intel_job_mapping.json b/src/main/resources/mappings/threat_intel_job_mapping.json index ffd165ae5..fbc1c03dc 100644 --- a/src/main/resources/mappings/threat_intel_job_mapping.json +++ b/src/main/resources/mappings/threat_intel_job_mapping.json @@ -1,9 +1,212 @@ { - "dynamic": "strict", + "dynamic": true, "_meta" : { - "schema_version": 1 + "schema_version": 2 }, "properties": { + "source_config": { + "properties": { + "version": { + "type": "long" + }, + "name": { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + }, + "format": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "created_by_user": { + "properties": { + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "backend_roles": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "roles": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "custom_attribute_names": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + } + } + }, + "created_at": { + "type": "date", + "format": "strict_date_time||epoch_millis" + }, + "source" : { + "properties": { + "s3": { + "properties": { + "bucket_name": { + "type": "keyword" + }, + "object_key": { + "type": "keyword" + }, + "region": { + "type": "keyword" + }, + "role_arn": { + "type": "keyword" + } + } + }, + "ioc_upload": { + "properties": { + "file_name": { + "type": "keyword" + }, + "iocs": { + "type" : "text" + } + } + } + } + }, + "enabled_time": { + "type": "date", + "format": "strict_date_time||epoch_millis" + }, + "last_update_time": { + "type": "date", + "format": "strict_date_time||epoch_millis" + }, + "schedule": { + "properties": { + "interval": { + "properties": { + "period": { + "type": "integer" + }, + "start_time": { + "type": "date", + "format": "strict_date_time||epoch_millis" + }, + "unit": { + "type": "keyword" + } + } + } + } + }, + "state": { + "type": "keyword" + }, + "refresh_type": { + "type": "keyword" + }, + "enabled_for_scan": { + "type": "boolean" + }, + "last_refreshed_time": { + "type": "date", + "format": "strict_date_time||epoch_millis" + }, + "last_refreshed_user": { + "properties": { + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "backend_roles": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "roles": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "custom_attribute_names": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + } + } + }, + "enabled": { + "type": "boolean" + }, + "ioc_store_config": { + "properties": { + "default": { + "properties": { + "ioc_to_index_details": { + "properties": { + "ioc_type": { + "type": "keyword" + }, + "index_pattern": { + "type": "keyword" + }, + "active_index": { + "type": "keyword" + } + } + } + } + } + } + }, + "ioc_types": { + "type": "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + } + } + }, "schema_version": { "type": "integer" }, diff --git a/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java b/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java index 47c33e138..d6294ee76 100644 --- a/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java +++ b/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java @@ -75,7 +75,7 @@ public void testCreateDetectorWithThreatIntelEnabled_updateDetectorWithThreatInt " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(2, response.getHits().getTotalHits().value); @@ -275,7 +275,7 @@ public void testCreateDetectorWithThreatIntelDisabled_updateDetectorWithThreatIn " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(1, response.getHits().getTotalHits().value); @@ -372,7 +372,7 @@ public void testCreateDetectorWithThreatIntelEnabledAndNoRules_triggerDetectionT " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(1, response.getHits().getTotalHits().value); @@ -466,7 +466,7 @@ public void testCreateDetectorWithThreatIntelEnabled_triggerDetectionTypeOnlyThr " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(1, response.getHits().getTotalHits().value); @@ -561,7 +561,7 @@ public void testCreateDetectorWithThreatIntelEnabled_triggerWithBothDetectionTyp " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(1, response.getHits().getTotalHits().value); @@ -653,7 +653,7 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithThreatIntelDete " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(1, response.getHits().getTotalHits().value); @@ -745,7 +745,7 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithRulesDetectionT " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(1, response.getHits().getTotalHits().value); @@ -802,4 +802,4 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithRulesDetectionT /** findings are present but alerts are NOT generated as detection type mentioned in trigger is threat_intel only but finding is from rules*/ Assert.assertEquals(3, getAlertsBody.get("total_alerts")); } -} +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/LogTypeServiceTests.java b/src/test/java/org/opensearch/securityanalytics/LogTypeServiceTests.java index 217fa0a03..cd467313b 100644 --- a/src/test/java/org/opensearch/securityanalytics/LogTypeServiceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/LogTypeServiceTests.java @@ -47,9 +47,9 @@ protected void beforeTest() throws Exception { List dummyLogTypes = List.of( new LogType(null, "test_logtype", "", true, List.of( - new LogType.Mapping("rawFld1", "ecsFld1", "ocsfFld1"), - new LogType.Mapping("rawFld2", "ecsFld2", "ocsfFld2"), - new LogType.Mapping("rawFld3", "ecsFld3", "ocsfFld3") + new LogType.Mapping("rawFld1", "ecsFld1", "ocsfFld1", "ocsf11Fld1"), + new LogType.Mapping("rawFld2", "ecsFld2", "ocsfFld2", "ocsf11Fld2"), + new LogType.Mapping("rawFld3", "ecsFld3", "ocsfFld3", "ocsf11Fld3") ), List.of(new LogType.IocFields("ip", List.of("dst.ip"))) ) diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index d2216415d..8bd62b34e 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -26,6 +26,7 @@ import org.opensearch.client.WarningsHandler; import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.metadata.MappingMetadata; +import org.opensearch.core.common.Strings; import org.opensearch.common.UUIDs; import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.Settings; @@ -34,6 +35,7 @@ import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.commons.ConfigConstants; import org.opensearch.commons.alerting.model.ScheduledJob; +import org.opensearch.commons.alerting.model.action.Action; import org.opensearch.commons.alerting.util.IndexUtilsKt; import org.opensearch.commons.rest.SecureRestClientBuilder; import org.opensearch.core.common.Strings; @@ -51,15 +53,27 @@ import org.opensearch.search.SearchHit; import org.opensearch.securityanalytics.action.AlertDto; import org.opensearch.securityanalytics.action.CreateIndexMappingsRequest; +import org.opensearch.securityanalytics.action.TestS3ConnectionRequest; import org.opensearch.securityanalytics.action.UpdateIndexMappingsRequest; import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; +import org.opensearch.securityanalytics.correlation.CorrelationEngineRestApiIT; import org.opensearch.securityanalytics.correlation.index.query.CorrelationQueryBuilder; import org.opensearch.securityanalytics.mapper.MappingsTraverser; +import org.opensearch.securityanalytics.model.CorrelationQuery; import org.opensearch.securityanalytics.model.CorrelationRule; +import org.opensearch.securityanalytics.model.CorrelationRuleTrigger; import org.opensearch.securityanalytics.model.CustomLogType; import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.securityanalytics.model.DetectorInput; +import org.opensearch.securityanalytics.model.DetectorRule; +import org.opensearch.securityanalytics.model.DetectorTrigger; import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; +import org.opensearch.securityanalytics.model.threatintel.ThreatIntelAlert; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto; import org.opensearch.securityanalytics.util.CorrelationIndices; import org.opensearch.test.rest.OpenSearchRestTestCase; @@ -89,7 +103,18 @@ import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.MAPPER_BASE_URI; import static org.opensearch.securityanalytics.TestHelpers.productIndexAvgAggRule; import static org.opensearch.securityanalytics.TestHelpers.sumAggregationTestRule; +import static org.opensearch.securityanalytics.TestHelpers.adLdapLogMappings; +import static org.opensearch.securityanalytics.TestHelpers.appLogMappings; +import static org.opensearch.securityanalytics.TestHelpers.productIndexAvgAggRule; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndTriggers; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndTriggersAndType; +import static org.opensearch.securityanalytics.TestHelpers.randomIndex; +import static org.opensearch.securityanalytics.TestHelpers.s3AccessLogMappings; +import static org.opensearch.securityanalytics.TestHelpers.sumAggregationTestRule; +import static org.opensearch.securityanalytics.TestHelpers.vpcFlowMappings; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; +import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.IOC_ALL_INDEX_PATTERN; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_INDEX_MAX_AGE; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_RETENTION_PERIOD; @@ -98,7 +123,7 @@ import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_ROLLOVER_PERIOD; -import static org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataUtils.getTifdList; +import static org.opensearch.securityanalytics.threatIntel.util.ThreatIntelFeedDataUtils.getTifdList; import static org.opensearch.securityanalytics.util.RuleTopicIndices.ruleTopicIndexSettings; public class SecurityAnalyticsRestTestCase extends OpenSearchRestTestCase { @@ -135,7 +160,8 @@ protected void createRuleTopicIndex(String detectorType, String additionalMappin assertEquals(RestStatus.OK, restStatus(response)); } } - protected void verifyWorkflow(Map detectorMap, List monitorIds, int expectedDelegatesNum) throws IOException{ + + protected void verifyWorkflow(Map detectorMap, List monitorIds, int expectedDelegatesNum) throws IOException { String workflowId = ((List) detectorMap.get("workflow_ids")).get(0); Map workflow = searchWorkflow(workflowId); @@ -144,27 +170,27 @@ protected void verifyWorkflow(Map detectorMap, List moni List> workflowInputs = (List>) workflow.get("inputs"); assertEquals("Workflow not found", 1, workflowInputs.size()); - Map sequence = ((Map)((Map)workflowInputs.get(0).get("composite_input")).get("sequence")); + Map sequence = ((Map) ((Map) workflowInputs.get(0).get("composite_input")).get("sequence")); assertNotNull("Sequence is null", sequence); List> delegates = (List>) sequence.get("delegates"); assertEquals(expectedDelegatesNum, delegates.size()); // Assert that all monitors are present - for (Map delegate: delegates) { + for (Map delegate : delegates) { assertTrue("Monitor doesn't exist in monitor list", monitorIds.contains(delegate.get("monitor_id"))); } } - protected Map searchWorkflow(String workflowId) throws IOException{ - String workflowRequest = "{\n" + - " \"query\":{\n" + - " \"term\":{\n" + - " \"_id\":{\n" + - " \"value\":\"" + workflowId + "\"\n" + - " }\n" + - " }\n" + - " }\n" + - "}"; + protected Map searchWorkflow(String workflowId) throws IOException { + String workflowRequest = "{\n" + + " \"query\":{\n" + + " \"term\":{\n" + + " \"_id\":{\n" + + " \"value\":\"" + workflowId + "\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; List hits = executeWorkflowSearch("/_plugins/_alerting/monitors", workflowRequest); if (hits.size() == 0) { @@ -176,21 +202,21 @@ protected Map searchWorkflow(String workflowId) throws IOExcepti } - protected List> getAllWorkflows() throws IOException{ - String workflowRequest = "{\n" + - " \"query\":{\n" + - " \"exists\":{\n" + - " \"field\": \"workflow\"" + - " }\n" + - " }\n" + - " }"; + protected List> getAllWorkflows() throws IOException { + String workflowRequest = "{\n" + + " \"query\":{\n" + + " \"exists\":{\n" + + " \"field\": \"workflow\"" + + " }\n" + + " }\n" + + " }"; List hits = executeSearch(ScheduledJob.SCHEDULED_JOBS_INDEX, workflowRequest); if (hits.size() == 0) { return new ArrayList<>(); } List> result = new ArrayList<>(); - for (SearchHit hit: hits) { + for (SearchHit hit : hits) { result.add((Map) hit.getSourceAsMap().get("workflow")); } return result; @@ -202,21 +228,21 @@ protected String createDetector(Detector detector) throws IOException { Map responseBody = asMap(createResponse); - return responseBody.get("_id").toString(); + return responseBody.get("_id").toString(); } protected void deleteDetector(String detectorId) throws IOException { makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId, Collections.emptyMap(), null); } - protected List getAllComponentTemplates() throws IOException { + protected List getAllComponentTemplates() throws IOException { Response response = makeRequest(client(), "GET", "_component_template", Collections.emptyMap(), null); assertEquals(RestStatus.OK, restStatus(response)); Map responseBody = asMap(response); return (List) responseBody.get("component_templates"); } - protected List getAllComposableIndexTemplates() throws IOException { + protected List getAllComposableIndexTemplates() throws IOException { Response response = makeRequest(client(), "GET", "_index_template", Collections.emptyMap(), null); assertEquals(RestStatus.OK, restStatus(response)); Map responseBody = asMap(response); @@ -242,22 +268,22 @@ void setDebugLogLevel() throws IOException, InterruptedException { " }"); - makeRequest(client(), "PUT", "_cluster/settings", Collections.emptyMap(), se, new BasicHeader("Content-Type", "application/json")); + updateClusterSetting("plugins.security_analytics.enable_detectors_with_dedicated_query_indices", "true"); } protected final List clusterPermissions = List.of( - "cluster:admin/opensearch/securityanalytics/detector/*", - "cluster:admin/opendistro/alerting/alerts/*", - "cluster:admin/opendistro/alerting/findings/*", - "cluster:admin/opensearch/securityanalytics/mapping/*", - "cluster:admin/opensearch/securityanalytics/rule/*" + "cluster:admin/opensearch/securityanalytics/detector/*", + "cluster:admin/opendistro/alerting/alerts/*", + "cluster:admin/opendistro/alerting/findings/*", + "cluster:admin/opensearch/securityanalytics/mapping/*", + "cluster:admin/opensearch/securityanalytics/rule/*" ); protected final List indexPermissions = List.of( - "indices:admin/mappings/get", - "indices:admin/mapping/put", - "indices:data/read/search" + "indices:admin/mappings/get", + "indices:admin/mapping/put", + "indices:data/read/search" ); protected static String TEST_HR_ROLE = "hr_role"; @@ -292,7 +318,7 @@ protected String createTestIndex(RestClient client, String index, String mapping protected String createDocumentWithNFields(int numOfFields) { StringBuilder doc = new StringBuilder(); doc.append("{"); - for(int i = 0; i < numOfFields - 1; i++) { + for (int i = 0; i < numOfFields - 1; i++) { doc.append("\"id").append(i).append("\": 5,"); } doc.append("\"last_field\": 100 }"); @@ -306,7 +332,7 @@ protected Response makeRequest(RestClient client, String method, String endpoint RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder(); options.setWarningsHandler(WarningsHandler.PERMISSIVE); - for (Header header: headers) { + for (Header header : headers) { options.addHeader(header.getName(), header.getValue()); } request.setOptions(options.build()); @@ -482,7 +508,7 @@ protected String createDestination() throws IOException { protected void createAlertingMonitorConfigIndex(String mapping) throws IOException { if (!doesIndexExist(ScheduledJob.SCHEDULED_JOBS_INDEX)) { - String mappingHack = mapping == null? alertingScheduledJobMappings(): mapping; + String mappingHack = mapping == null ? alertingScheduledJobMappings() : mapping; Settings settings = Settings.builder().put("index.hidden", true).build(); createTestIndex(ScheduledJob.SCHEDULED_JOBS_INDEX, mappingHack, settings); } @@ -505,13 +531,13 @@ protected List getRandomPrePackagedRules() throws IOException { ); } - protected List createAggregationRules () throws IOException { + protected List createAggregationRules() throws IOException { return new ArrayList<>(Arrays.asList(createRule(productIndexAvgAggRule()), createRule(sumAggregationTestRule()))); } protected String createRule(String rule) throws IOException { Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "test_windows"), - new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); return responseBody.get("_id").toString(); @@ -550,7 +576,7 @@ protected Response indexDoc(String index, String id, String doc) throws IOExcept protected Response indexDoc(RestClient client, String index, String id, String doc, Boolean refresh) throws IOException { StringEntity requestBody = new StringEntity(doc, ContentType.APPLICATION_JSON); - Map params = refresh? Map.of("refresh", "true"): Collections.emptyMap(); + Map params = refresh ? Map.of("refresh", "true") : Collections.emptyMap(); Response response = makeRequest(client, "POST", String.format(Locale.getDefault(), "%s/_doc/%s?op_type=create", index, id), params, requestBody); Assert.assertTrue(String.format(Locale.getDefault(), "Unable to index doc: '%s...' to index: '%s'", doc.substring(0, 15), index), List.of(RestStatus.OK, RestStatus.CREATED).contains(restStatus(response))); return response; @@ -599,7 +625,7 @@ public Response searchAlertingFindings(Map params) throws IOExce baseEndpoint += "?"; } - for (Map.Entry param: params.entrySet()) { + for (Map.Entry param : params.entrySet()) { baseEndpoint += String.format(Locale.getDefault(), "%s=%s&", param.getKey(), param.getValue()); } @@ -629,9 +655,9 @@ public static SearchResponse executeSearchRequest(RestClient client, String inde Response response = client.performRequest(request); XContentParser parser = JsonXContent.jsonXContent.createParser( - new NamedXContentRegistry(ClusterModule.getNamedXWriteables()), - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, - response.getEntity().getContent() + new NamedXContentRegistry(ClusterModule.getNamedXWriteables()), + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + response.getEntity().getContent() ); return SearchResponse.fromXContent(parser); } @@ -660,6 +686,22 @@ protected HttpEntity toHttpEntity(CorrelationRule rule) throws IOException { return new StringEntity(toJsonString(rule), ContentType.APPLICATION_JSON); } + protected HttpEntity toHttpEntity(SATIFSourceConfigDto saTifSourceConfigDto) throws IOException { + return new StringEntity(toJsonString(saTifSourceConfigDto), ContentType.APPLICATION_JSON); + } + + protected HttpEntity toHttpEntity(IocFinding iocFinding) throws IOException { + return new StringEntity(toJsonString(iocFinding), ContentType.APPLICATION_JSON); + } + + protected HttpEntity toHttpEntity(ThreatIntelMonitorDto threatIntelMonitorDto) throws IOException { + return new StringEntity(toJsonString(threatIntelMonitorDto), ContentType.APPLICATION_JSON); + } + + protected HttpEntity toHttpEntity(TestS3ConnectionRequest testS3ConnectionRequest) throws IOException { + return new StringEntity(toJsonString(testS3ConnectionRequest), ContentType.APPLICATION_JSON); + } + protected RestStatus restStatus(Response response) { return RestStatus.fromCode(response.getStatusLine().getStatusCode()); } @@ -703,6 +745,31 @@ protected String toJsonString(ThreatIntelFeedData tifd) throws IOException { return IndexUtilsKt.string(shuffleXContent(tifd.toXContent(builder, ToXContent.EMPTY_PARAMS))); } + private String toJsonString(SATIFSourceConfigDto saTifSourceConfigDto) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + return IndexUtilsKt.string(shuffleXContent(saTifSourceConfigDto.toXContent(builder, ToXContent.EMPTY_PARAMS))); + } + + private String toJsonString(ThreatIntelMonitorDto threatIntelMonitorDto) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + return IndexUtilsKt.string(shuffleXContent(threatIntelMonitorDto.toXContent(builder, ToXContent.EMPTY_PARAMS))); + } + + private String toJsonString(IocFinding iocFinding) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + return IndexUtilsKt.string(shuffleXContent(iocFinding.toXContent(builder, ToXContent.EMPTY_PARAMS))); + } + + public String toJsonString(ThreatIntelAlert alert) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + return IndexUtilsKt.string(shuffleXContent(alert.toXContent(builder, ToXContent.EMPTY_PARAMS))); + } + + private String toJsonString(TestS3ConnectionRequest testS3ConnectionRequest) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + return IndexUtilsKt.string(shuffleXContent(testS3ConnectionRequest.toXContent(builder, ToXContent.EMPTY_PARAMS))); + } + private String alertingScheduledJobMappings() { return " \"_meta\" : {\n" + " \"schema_version\": 5\n" + @@ -1246,33 +1313,28 @@ protected Settings restAdminSettings() { } - @Override - protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException - { + protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException { if (securityEnabled()) { String keystore = settings.get(ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_FILEPATH); - if (keystore != null) { + if (keystore != null) { // create adminDN (super-admin) client //log.info("keystore not null"); URI uri = null; try { uri = SecurityAnalyticsRestTestCase.class.getClassLoader().getResource("sample.pem").toURI(); - } - catch(URISyntaxException e) { + } catch (URISyntaxException e) { return null; } Path configPath = PathUtils.get(uri).getParent().toAbsolutePath(); return new SecureRestClientBuilder(settings, configPath).setSocketTimeout(60000).build(); - } - else { + } else { // create client with passed user String userName = System.getProperty("user"); String password = System.getProperty("password"); return new SecureRestClientBuilder(hosts, isHttps(), userName, password).setSocketTimeout(60000).build(); } - } - else { + } else { RestClientBuilder builder = RestClient.builder(hosts); configureClient(builder, settings); builder.setStrictDeprecationMode(true); @@ -1289,7 +1351,7 @@ protected void createIndexRole(String name, List clusterPermissions, Lis response = ex.getResponse(); } // Role already exists - if(response.getStatusLine().getStatusCode() == RestStatus.OK.getStatus()) { + if (response.getStatusLine().getStatusCode() == RestStatus.OK.getStatus()) { return; } @@ -1299,19 +1361,19 @@ protected void createIndexRole(String name, List clusterPermissions, Lis String indexPatternsStr = indexPatterns.stream().map(p -> "\"" + p + "\"").collect(Collectors.joining(",")); String entity = "{\n" + - "\"cluster_permissions\": [\n" + - "" + clusterPermissionsStr + "\n" + - "], \n" + - "\"index_permissions\": [\n" + + "\"cluster_permissions\": [\n" + + "" + clusterPermissionsStr + "\n" + + "], \n" + + "\"index_permissions\": [\n" + "{" + - "\"fls\": [], " + - "\"masked_fields\": [], " + - "\"allowed_actions\": [" + indexPermissionsStr + "], " + - "\"index_patterns\": [" + indexPatternsStr + "]" + + "\"fls\": [], " + + "\"masked_fields\": [], " + + "\"allowed_actions\": [" + indexPermissionsStr + "], " + + "\"index_patterns\": [" + indexPatternsStr + "]" + "}" + - "], " + - "\"tenant_permissions\": []" + - "}"; + "], " + + "\"tenant_permissions\": []" + + "}"; request.setJsonEntity(entity); client().performRequest(request); @@ -1328,7 +1390,7 @@ protected void createCustomRole(String name, String clusterPermissions) throws I client().performRequest(request); } - public void createUser(String name, String[] backendRoles) throws IOException { + public void createUser(String name, String[] backendRoles) throws IOException { Request request = new Request("PUT", String.format(Locale.getDefault(), "/_plugins/_security/api/internalusers/%s", name)); String broles = String.join(",", backendRoles); //String roles = String.join(",", customRoles); @@ -1341,9 +1403,9 @@ public void createUser(String name, String[] backendRoles) throws IOException { client().performRequest(request); } - protected void createUserRolesMapping(String role, String[] users) throws IOException { + protected void createUserRolesMapping(String role, String[] users) throws IOException { Request request = new Request("PUT", String.format(Locale.getDefault(), "/_plugins/_security/api/rolesmapping/%s", role)); - String usersArr= String.join(",", users); + String usersArr = String.join(",", users); String entity = "{\n" + " \"backend_roles\" : [ ],\n" + " \"hosts\" : [ ],\n" + @@ -1353,34 +1415,34 @@ protected void createUserRolesMapping(String role, String[] users) throws IOExc client().performRequest(request); } - protected void enableOrDisableFilterBy(String trueOrFalse) throws IOException { + protected void enableOrDisableFilterBy(String trueOrFalse) throws IOException { Request request = new Request("PUT", "_cluster/settings"); String entity = "{\"persistent\":{\"plugins.security_analytics.filter_by_backend_roles\" : " + trueOrFalse + "}}"; request.setJsonEntity(entity); client().performRequest(request); } - protected void createUserWithDataAndCustomRole(String userName, String userPasswd, String roleName, String[] backendRoles, String clusterPermissions ) throws IOException { + protected void createUserWithDataAndCustomRole(String userName, String userPasswd, String roleName, String[] backendRoles, String clusterPermissions) throws IOException { String[] users = {userName}; createUser(userName, backendRoles); createCustomRole(roleName, clusterPermissions); createUserRolesMapping(roleName, users); } - protected void createUserWithDataAndCustomRole(String userName, String userPasswd, String roleName, String[] backendRoles, List clusterPermissions, List indexPermissions, List indexPatterns) throws IOException { + protected void createUserWithDataAndCustomRole(String userName, String userPasswd, String roleName, String[] backendRoles, List clusterPermissions, List indexPermissions, List indexPatterns) throws IOException { String[] users = {userName}; createUser(userName, backendRoles); createIndexRole(roleName, clusterPermissions, indexPermissions, indexPatterns); createUserRolesMapping(roleName, users); } - protected void createUserWithData(String userName, String userPasswd, String roleName, String[] backendRoles ) throws IOException { + protected void createUserWithData(String userName, String userPasswd, String roleName, String[] backendRoles) throws IOException { String[] users = {userName}; createUser(userName, backendRoles); createUserRolesMapping(roleName, users); } - public void createUserWithTestData(String user, String index, String role, String [] backendRoles, List indexPermissions) throws IOException{ + public void createUserWithTestData(String user, String index, String role, String[] backendRoles, List indexPermissions) throws IOException { String[] users = {user}; createUser(user, backendRoles); createTestIndex(client(), index, windowsIndexMapping(), Settings.EMPTY); @@ -1393,7 +1455,7 @@ protected void deleteUser(String name) throws IOException { client().performRequest(request); } - protected void tryDeletingRole(String name) throws IOException{ + protected void tryDeletingRole(String name) throws IOException { Response response; try { response = client().performRequest(new Request("GET", String.format(Locale.getDefault(), "/_plugins/_security/api/roles/%s", name))); @@ -1401,7 +1463,7 @@ protected void tryDeletingRole(String name) throws IOException{ response = ex.getResponse(); } // Role already exists - if(response.getStatusLine().getStatusCode() == RestStatus.OK.getStatus()) { + if (response.getStatusLine().getStatusCode() == RestStatus.OK.getStatus()) { Request request = new Request("DELETE", String.format(Locale.getDefault(), "/_plugins/_security/api/roles/%s", name)); client().performRequest(request); } @@ -1417,7 +1479,7 @@ boolean preserveODFEIndicesAfterTest() { } @After - protected void wipeAllODFEIndices() throws IOException { + protected void wipeAllODFEIndices() throws IOException { if (preserveODFEIndicesAfterTest()) return; Response response = client().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all")); @@ -1446,7 +1508,6 @@ protected void wipeAllODFEIndices() throws IOException { } - public List getAlertIndices(String detectorType) throws IOException { Response response = client().performRequest(new Request("GET", "/_cat/indices/" + DetectorMonitorConfig.getAllAlertsIndicesPattern(detectorType) + "?format=json")); XContentParser xcp = createParser(XContentType.JSON.xContent(), response.getEntity().getContent()); @@ -1455,11 +1516,47 @@ public List getAlertIndices(String detectorType) throws IOException { for (Object o : responseList) { if (o instanceof Map) { ((Map) o).forEach((BiConsumer) - (o1, o2) -> { - if (o1.equals("index")) { - indices.add((String) o2); - } - }); + (o1, o2) -> { + if (o1.equals("index")) { + indices.add((String) o2); + } + }); + } + } + return indices; + } + + public List getIocFindingIndices() throws IOException { + Response response = client().performRequest(new Request("GET", "/_cat/indices/" + IocFindingService.IOC_FINDING_INDEX_PATTERN_REGEXP + "?format=json")); + XContentParser xcp = createParser(XContentType.JSON.xContent(), response.getEntity().getContent()); + List responseList = xcp.list(); + List indices = new ArrayList<>(); + for (Object o : responseList) { + if (o instanceof Map) { + ((Map) o).forEach((BiConsumer) + (o1, o2) -> { + if (o1.equals("index")) { + indices.add((String) o2); + } + }); + } + } + return indices; + } + + public List getIocIndices() throws IOException { + Response response = client().performRequest(new Request("GET", "/_cat/indices/" + IOC_ALL_INDEX_PATTERN + "?format=json")); + XContentParser xcp = createParser(XContentType.JSON.xContent(), response.getEntity().getContent()); + List responseList = xcp.list(); + List indices = new ArrayList<>(); + for (Object o : responseList) { + if (o instanceof Map) { + ((Map) o).forEach((BiConsumer) + (o1, o2) -> { + if (o1.equals("index")) { + indices.add((String) o2); + } + }); } } return indices; @@ -1527,7 +1624,7 @@ public void updateClusterSetting(String setting, String value) throws IOExceptio " }" + "}"; settingJson = String.format(settingJson, setting, value); - makeRequest(client(), "PUT", "_cluster/settings", Collections.emptyMap(), new StringEntity(settingJson, ContentType.APPLICATION_JSON), new BasicHeader("Content-Type", "application/json")); + makeRequest(client(), "PUT", "_cluster/settings", Collections.emptyMap(), new StringEntity(settingJson, ContentType.APPLICATION_JSON), new BasicHeader("Content-Type", "application/json")); } public void acknowledgeAlert(String alertId, String detectorId) throws IOException { @@ -1561,7 +1658,7 @@ protected void createNetflowLogIndex(String indexName) throws IOException { " \"netflow.source_transport_port\": {" + " \"type\": \"integer\"" + " }" + - " }"; + " }"; createIndex(indexName, Settings.EMPTY, indexMapping); @@ -1579,18 +1676,18 @@ protected void createNetflowLogIndex(String indexName) throws IOException { Response response = client().performRequest(indexRequest); assertEquals(HttpStatus.SC_CREATED, response.getStatusLine().getStatusCode()); // Refresh everything - response = client().performRequest(new Request("POST", "_refresh")); - assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + //response = client().performRequest(new Request("POST", "_refresh")); + //assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } private Map getIndexAPI(String index) throws IOException { - Response resp = makeRequest(client(), "GET", "/" + index + "?expand_wildcards=all", Collections.emptyMap(), null); + Response resp = makeRequest(client(), "GET", "/" + index + "?expand_wildcards=all", Collections.emptyMap(), null); return asMap(resp); } private Map getIndexSettingsAPI(String index) throws IOException { - Response resp = makeRequest(client(), "GET", "/" + index + "/_settings?expand_wildcards=all", Collections.emptyMap(), null); + Response resp = makeRequest(client(), "GET", "/" + index + "/_settings?expand_wildcards=all", Collections.emptyMap(), null); Map respMap = asMap(resp); return respMap; } @@ -1631,8 +1728,9 @@ protected void createComposableIndexTemplate(String templateName, List i indexPatterns.stream().collect( Collectors.joining(",", "\"", "\"")) + "]," + - (componentTemplateName == null ? ("\"template\": {\"mappings\": {" + mappings + "}},") : "") + - (componentTemplateName != null ? ("\"composed_of\": [\"" + componentTemplateName + "\"],") : "") + + (componentTemplateName == null ? ("\"template\": {\"mappings\": {" + mappings + "}},") : "") + + (componentTemplateName != null ? ("\"composed_of\": [\"" + componentTemplateName + "\"],\"template\": {" + + "\"settings\": {\"index\":{\"mapping\":{\"total_fields\":{\"limit\":\"5000\"}},\"number_of_shards\":\"18\",\"number_of_replicas\":\"1\"}}},") : "") + "\"priority\":" + priority + "}"; Response response = makeRequest( @@ -1677,7 +1775,6 @@ protected Map getIndexMappingsSAFlat(String indexName) throws IO } - protected void createMappingsAPI(String indexName, String topicName) throws IOException { Request request = new Request("POST", MAPPER_BASE_URI); // both req params and req body are supported @@ -1714,6 +1811,14 @@ protected void createDatastreamAPI(String datastreamName) throws IOException { assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } + protected void createIndexAliasApi(String indexAlias, String indexName) throws IOException { + Request request = new Request("POST", "_aliases"); + request.setJsonEntity("{\"actions\":[{\"add\":{\"index\":\"" + indexName + "\",\"alias\":\"" + indexAlias + "\", " + + "\"is_write_index\": true}}]}"); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + } + protected void deleteDatastreamAPI(String datastreamName) throws IOException { Request request = new Request("DELETE", "_data_stream/" + datastreamName); @@ -1756,6 +1861,34 @@ protected void createSampleDatastream(String datastreamName, String mappings, bo createDatastreamAPI(datastreamName); } + protected void createSampleIndexTemplate(String indexPattern, String mappings, boolean useComponentTemplate) throws IOException { + String indexName = indexPattern.substring(0, indexPattern.length() - 1); + String componentTemplateMappings = "\"properties\": {" + + " \"netflow.destination_transport_port\":{ \"type\": \"long\" }," + + " \"netflow.destination_ipv4_address\":{ \"type\": \"ip\" }" + + "}"; + + if (mappings != null) { + componentTemplateMappings = mappings; + } + + if (useComponentTemplate) { + // Setup index_template + createComponentTemplateWithMappings( + "my_ds_component_template-" + indexName, + componentTemplateMappings + ); + } + createComposableIndexTemplate( + "my_index_template_ds-" + indexName, + List.of(indexPattern), + useComponentTemplate ? "my_ds_component_template-" + indexName : null, + mappings, + false, + 2 + ); + } + protected void restoreAlertsFindingsIMSettings() throws IOException { updateClusterSetting(ALERT_HISTORY_ROLLOVER_PERIOD.getKey(), "720m"); updateClusterSetting(ALERT_HISTORY_MAX_DOCS.getKey(), "100000"); @@ -1769,7 +1902,7 @@ protected void restoreAlertsFindingsIMSettings() throws IOException { } - protected void enableOrDisableWorkflow(String trueOrFalse) throws IOException { + protected void enableOrDisableWorkflow(String trueOrFalse) throws IOException { Request request = new Request("PUT", "_cluster/settings"); String entity = "{\"persistent\":{\"plugins.security_analytics.filter_by_backend_roles\" : " + trueOrFalse + "}}"; request.setJsonEntity(entity); @@ -1792,6 +1925,329 @@ public String getMatchAllSearchRequestString(int num) { "}"; } + protected CorrelationEngineRestApiIT.LogIndices createIndices() throws IOException { + CorrelationEngineRestApiIT.LogIndices indices = new CorrelationEngineRestApiIT.LogIndices(); + indices.adLdapLogsIndex = createTestIndex("ad_logs", adLdapLogMappings()); + indices.s3AccessLogsIndex = createTestIndex("s3_access_logs", s3AccessLogMappings()); + indices.appLogsIndex = createTestIndex("app_logs", appLogMappings()); + indices.windowsIndex = createTestIndex(randomIndex(), windowsIndexMapping()); + indices.vpcFlowsIndex = createTestIndex("vpc_flow", vpcFlowMappings()); + return indices; + } + + protected String createNetworkToWindowsFieldBasedRule(CorrelationEngineRestApiIT.LogIndices indices) throws IOException { + CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, null, "network", "srcaddr"); + CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, null, "test_windows", "SourceIp"); + + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to windows", List.of(query1, query4), 300000L, null); + Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); + request.setJsonEntity(toJsonString(rule)); + Response response = client().performRequest(request); + + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + return entityAsMap(response).get("_id").toString(); + } + + protected String createNetworkToWindowsFilterQueryBasedRule(LogIndices indices) throws IOException { + CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, "srcaddr:1.2.3.4", "network", null); + CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, "SourceIp:1.2.3.4", "test_windows", null); + + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to windows", List.of(query1, query4), 300000L, null); + Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); + request.setJsonEntity(toJsonString(rule)); + Response response = client().performRequest(request); + + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + return entityAsMap(response).get("_id").toString(); + } + + protected String createNetworkToCustomLogTypeFieldBasedRule(LogIndices indices, String customLogTypeName, String customLogTypeIndex) throws IOException { + CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, null, "network", "srcaddr"); + CorrelationQuery query4 = new CorrelationQuery(customLogTypeIndex, null, customLogTypeName, "SourceIp"); + + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to custom log type", List.of(query1, query4), 300000L, null); + Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); + request.setJsonEntity(toJsonString(rule)); + Response response = client().performRequest(request); + + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + return entityAsMap(response).get("_id").toString(); + } + + protected String createNetworkToAdLdapToWindowsRule(LogIndices indices) throws IOException { + CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, "dstaddr:4.5.6.7", "network", null); + CorrelationQuery query2 = new CorrelationQuery(indices.adLdapLogsIndex, "ResultType:50126", "ad_ldap", null); + CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, "Domain:NTAUTHORI*", "test_windows", null); + + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to ad_ldap to windows", List.of(query1, query2, query4), 300000L, null); + Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); + request.setJsonEntity(toJsonString(rule)); + Response response = client().performRequest(request); + + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + return entityAsMap(response).get("_id").toString(); + } + + protected String createNetworkToAdLdapToWindowsRuleWithTrigger(LogIndices indices) throws IOException { + CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, "dstaddr:4.5.6.7", "network", null); + CorrelationQuery query2 = new CorrelationQuery(indices.adLdapLogsIndex, "ResultType:50126", "ad_ldap", null); + CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, "Domain:NTAUTHORI*", "test_windows", null); + List actions = new ArrayList<>(); + CorrelationRuleTrigger trigger = new CorrelationRuleTrigger("trigger-123", "Trigger 1", "high", actions); + + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to ad_ldap to windows", List.of(query1, query2, query4), 300000L, trigger); + Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); + request.setJsonEntity(toJsonString(rule)); + Response response = client().performRequest(request); + + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + return entityAsMap(response).get("_id").toString(); + } + + protected String createWindowsToAppLogsToS3LogsRule(LogIndices indices) throws IOException { + CorrelationQuery query1 = new CorrelationQuery(indices.windowsIndex, "HostName:EC2AMAZ*", "test_windows", null); + CorrelationQuery query2 = new CorrelationQuery(indices.appLogsIndex, "endpoint:\\/customer_records.txt", "others_application", null); + CorrelationQuery query4 = new CorrelationQuery(indices.s3AccessLogsIndex, "aws.cloudtrail.eventName:ReplicateObject", "s3", null); + + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "windows to app_logs to s3 logs", List.of(query1, query2, query4), 300000L, null); + Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); + request.setJsonEntity(toJsonString(rule)); + Response response = client().performRequest(request); + + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + return entityAsMap(response).get("_id").toString(); + } + + protected String createCloudtrailFieldBasedRule(String index, String field, Long timeWindow) throws IOException { + CorrelationQuery query1 = new CorrelationQuery(index, "EventName:CreateUser", "cloudtrail", field); + CorrelationQuery query2 = new CorrelationQuery(index, "EventName:DeleteUser", "cloudtrail", field); + + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "cloudtrail field based", List.of(query1, query2), timeWindow, null); + Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); + request.setJsonEntity(toJsonString(rule)); + Response response = client().performRequest(request); + + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + return entityAsMap(response).get("_id").toString(); + } + + protected String createCloudtrailFieldBasedRuleWithTrigger(String index, String field, Long timeWindow) throws IOException { + CorrelationQuery query1 = new CorrelationQuery(index, "EventName:CreateUser", "cloudtrail", field); + CorrelationQuery query2 = new CorrelationQuery(index, "EventName:DeleteUser", "cloudtrail", field); + List actions = new ArrayList<>(); + CorrelationRuleTrigger trigger = new CorrelationRuleTrigger("trigger-345", "Trigger 2", "high", actions); + CorrelationRule rule = new CorrelationRule("correlation-rule-1", CorrelationRule.NO_VERSION, "cloudtrail field based", List.of(query1, query2), timeWindow, trigger); + Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); + request.setJsonEntity(toJsonString(rule)); + Response response = client().performRequest(request); + + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + return entityAsMap(response).get("_id").toString(); + } + + @SuppressWarnings("unchecked") + protected String createVpcFlowDetector(String indexName) throws IOException { + Detector vpcFlowDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("vpc flow detector for security analytics", List.of(indexName), List.of(), + getPrePackagedRules("network").stream().map(DetectorRule::new).collect(Collectors.toList()))), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("network"), List.of(), List.of(), List.of(), List.of(), List.of())), "network"); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(vpcFlowDetector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String createdId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + return ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + } + + @SuppressWarnings("unchecked") + protected String createAdLdapDetector(String indexName) throws IOException { + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{\n" + + " \"index_name\": \"" + indexName + "\",\n" + + " \"rule_topic\": \"ad_ldap\",\n" + + " \"partial\": true,\n" + + " \"alias_mappings\": {\n" + + " \"properties\": {\n" + + " \"azure.signinlogs.properties.user_id\": {\n" + + " \"path\": \"azure.signinlogs.props.user_id\",\n" + + " \"type\": \"alias\"\n" + + " },\n" + + " \"azure-platformlogs-result_type\": {\n" + + " \"path\": \"azure.platformlogs.result_type\",\n" + + " \"type\": \"alias\"\n" + + " },\n" + + " \"azure-signinlogs-result_description\": {\n" + + " \"path\": \"azure.signinlogs.result_description\",\n" + + " \"type\": \"alias\"\n" + + " },\n" + + " \"timestamp\": {\n" + + " \"path\": \"creationTime\",\n" + + " \"type\": \"alias\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(RestStatus.OK.getStatus(), response.getStatusLine().getStatusCode()); + + Detector adLdapDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("ad_ldap logs detector for security analytics", List.of(indexName), List.of(), + getPrePackagedRules("ad_ldap").stream().map(DetectorRule::new).collect(Collectors.toList()))), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("ad_ldap"), List.of(), List.of(), List.of(), List.of(), List.of())), "ad_ldap"); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(adLdapDetector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String createdId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + return ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + } + + @SuppressWarnings("unchecked") + protected String createTestWindowsDetector(String indexName) throws IOException { + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + indexName + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(RestStatus.OK.getStatus(), response.getStatusLine().getStatusCode()); + + Detector windowsDetector = randomDetectorWithInputsAndTriggers(List.of(new DetectorInput("windows detector for security analytics", List.of(indexName), List.of(), + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()))), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(windowsDetector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String createdId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + return ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + } + + @SuppressWarnings("unchecked") + protected String createAppLogsDetector(String indexName) throws IOException { + Detector appLogsDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("app logs detector for security analytics", List.of(indexName), List.of(), + getPrePackagedRules("others_application").stream().map(DetectorRule::new).collect(Collectors.toList()))), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("others_application"), List.of(), List.of(), List.of(), List.of(), List.of())), "others_application"); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(appLogsDetector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String createdId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + return ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + } + + @SuppressWarnings("unchecked") + protected String createS3Detector(String indexName) throws IOException { + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{\n" + + " \"index_name\": \"s3_access_logs\",\n" + + " \"rule_topic\": \"s3\",\n" + + " \"partial\": true,\n" + + " \"alias_mappings\": {\n" + + " \"properties\": {\n" + + " \"aws-cloudtrail-event_source\": {\n" + + " \"type\": \"alias\",\n" + + " \"path\": \"aws.cloudtrail.event_source\"\n" + + " },\n" + + " \"aws.cloudtrail.event_name\": {\n" + + " \"type\": \"alias\",\n" + + " \"path\": \"aws.cloudtrail.event_name\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(RestStatus.OK.getStatus(), response.getStatusLine().getStatusCode()); + + Detector s3AccessLogsDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("s3 access logs detector for security analytics", List.of(indexName), List.of(), + getPrePackagedRules("s3").stream().map(DetectorRule::new).collect(Collectors.toList()))), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("s3"), List.of(), List.of(), List.of(), List.of(), List.of())), "s3"); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(s3AccessLogsDetector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String createdId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + return ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + } + /** * We need to be able to dump the jacoco coverage before cluster is shut down. * The new internal testing framework removed some of the gradle tasks we were listening to @@ -1830,4 +2286,63 @@ public static void dumpCoverage() throws IOException, MalformedObjectNameExcepti throw new RuntimeException("Failed to dump coverage: " + ex); } } -} \ No newline at end of file + + protected Map> createTestAlias( + String alias, int numOfAliasIndices, boolean includeWriteIndex + ) throws IOException { + return createTestAlias( + alias, + randomAliasIndices(alias, numOfAliasIndices, includeWriteIndex), + true + ); + } + + protected Map> createTestAlias( + String alias, Map indices, boolean createIndices) throws IOException { + Map indicesMap = new java.util.HashMap<>(indices); + Map> result = new java.util.HashMap<>(); + XContentBuilder indicesJson = XContentFactory.jsonBuilder() + .startObject() + .startArray("actions"); + for (Map.Entry entry : indicesMap.entrySet()) { + if (createIndices) + createTestIndex(entry.getKey(), windowsIndexMapping()); + boolean isWriteIndex = entry.getValue(); + indicesJson.startObject() + .startObject("add") + .field("index", entry.getKey()) + .field("alias", alias) + .field("is_write_index", isWriteIndex) + .endObject() + .endObject(); + } + indicesJson.endArray().endObject(); + makeRequest(client(), "POST", "/_aliases", Collections.emptyMap(), new StringEntity(indicesJson.toString(), ContentType.APPLICATION_JSON)); + result.put(alias, indicesMap); + return result; + } + + + protected static Map randomAliasIndices( + String alias, int num, boolean includeWriteIndex) { + Map indices = new HashMap<>(); + int writeIndex = randomIntBetween(0, num - 1); + for (int i = 0; i < num; i++) { + String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + while (indexName.equals(alias) || indices.containsKey(indexName)) { + indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + } + boolean isWriteIndex = includeWriteIndex && i == writeIndex; + indices.put(indexName, isWriteIndex); + } + return indices; + } + + public static class LogIndices { + public String vpcFlowsIndex; + public String adLdapLogsIndex; + public String windowsIndex; + public String appLogsIndex; + public String s3AccessLogsIndex; + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index ad50a0dde..2d826eba1 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -6,7 +6,6 @@ import com.carrotsearch.randomizedtesting.generators.RandomNumbers; import org.apache.lucene.tests.util.LuceneTestCase; -import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; @@ -19,16 +18,30 @@ import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.script.Script; import org.opensearch.script.ScriptType; +import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.model.CorrelationQuery; import org.opensearch.securityanalytics.model.CorrelationRule; +import org.opensearch.securityanalytics.model.CorrelationRuleTrigger; import org.opensearch.securityanalytics.model.CustomLogType; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; +import org.opensearch.securityanalytics.model.threatintel.ThreatIntelAlert; +import org.opensearch.securityanalytics.threatIntel.common.RefreshType; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; +import org.opensearch.securityanalytics.threatIntel.model.IocStoreConfig; +import org.opensearch.securityanalytics.threatIntel.model.S3Source; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.model.Source; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.OpenSearchRestTestCase; @@ -78,25 +91,28 @@ public static Detector randomDetectorWithInputsAndThreatIntelAndTriggers(List inputs, List triggers) { return randomDetector(null, null, null, inputs, triggers, null, null, null, null, false); } + public static Detector randomDetectorWithInputs(List inputs, String detectorType) { return randomDetector(null, detectorType, null, inputs, List.of(), null, null, null, null, false); } - public static Detector randomDetectorWithTriggers(List triggers) { return randomDetector(null, null, null, List.of(), triggers, null, null, null, null, false); } + public static Detector randomDetectorWithTriggers(List rules, List triggers) { DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), Collections.emptyList(), rules.stream().map(DetectorRule::new).collect(Collectors.toList())); return randomDetector(null, null, null, List.of(input), triggers, null, null, null, null, false); } + public static Detector randomDetectorWithTriggers(List rules, List triggers, List inputIndices) { DetectorInput input = new DetectorInput("windows detector for security analytics", inputIndices, Collections.emptyList(), rules.stream().map(DetectorRule::new).collect(Collectors.toList())); return randomDetector(null, null, null, List.of(input), triggers, null, true, null, null, false); } + public static Detector randomDetectorWithTriggersAndScheduleAndEnabled(List rules, List triggers, Schedule schedule, boolean enabled) { DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), Collections.emptyList(), rules.stream().map(DetectorRule::new).collect(Collectors.toList())); @@ -197,32 +213,32 @@ public static Detector randomDetectorWithNoUser() { Instant lastUpdateTime = Instant.now().truncatedTo(ChronoUnit.MILLIS); return new Detector( - null, - null, - name, - enabled, - schedule, - lastUpdateTime, - enabledTime, - detectorType, - null, - inputs, - Collections.emptyList(), - Collections.singletonList(""), - "", - "", - "", - "", - "", - "", - Collections.emptyMap(), - Collections.emptyList(), - false + null, + null, + name, + enabled, + schedule, + lastUpdateTime, + enabledTime, + detectorType, + null, + inputs, + Collections.emptyList(), + Collections.singletonList(""), + "", + "", + "", + "", + "", + "", + Collections.emptyMap(), + Collections.emptyList(), + false ); } public static CorrelationRule randomCorrelationRule(String name) { - name = name.isEmpty()? ">": name; + name = name.isEmpty() ? ">" : name; return new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, name, List.of( new CorrelationQuery("vpc_flow1", "dstaddr:192.168.1.*", "network", null), @@ -230,6 +246,17 @@ public static CorrelationRule randomCorrelationRule(String name) { ), 300000L, null); } + public static CorrelationRule randomCorrelationRuleWithTrigger(String name) { + name = name.isEmpty() ? ">" : name; + List actions = new ArrayList(); + CorrelationRuleTrigger trigger = new CorrelationRuleTrigger("trigger-123", "Trigger 1", "high", actions); + return new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, name, + List.of( + new CorrelationQuery("vpc_flow1", "dstaddr:192.168.1.*", "network", null), + new CorrelationQuery("ad_logs1", "azure.platformlogs.result_type:50126", "ad_ldap", null) + ), 300000L, trigger); + } + public static String randomRule() { return "title: Remote Encrypting File System Abuse\n" + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + @@ -320,8 +347,8 @@ public static String randomRuleWithNotCondition() { " - Legitimate usage of remote file encryption\n" + "level: high"; } - - public static String randomRuleWithCriticalSeverity() { + + public static String randomRuleWithCriticalSeverity() { return "title: Remote Encrypting File System Abuse\n" + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + @@ -458,7 +485,7 @@ public static String randomRuleForMappingView(String field) { " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + "detection:\n" + " selection:\n" + - " "+ field + ": 'ACL'\n" + + " " + field + ": 'ACL'\n" + " condition: selection\n" + "falsepositives:\n" + " - Legitimate usage of remote file encryption\n" + @@ -675,7 +702,7 @@ public static String productIndexMaxAggRule() { " condition: sel | max(fieldA) by fieldB > 110"; } - public static String randomProductDocument(){ + public static String randomProductDocument() { return "{\n" + " \"name\": \"laptop\",\n" + " \"fieldA\": 123,\n" + @@ -684,7 +711,7 @@ public static String randomProductDocument(){ "}\n"; } - public static String randomProductDocumentWithTime(long time){ + public static String randomProductDocumentWithTime(long time) { return "{\n" + " \"fieldA\": 123,\n" + " \"mappedB\": 111,\n" + @@ -799,6 +826,18 @@ public static String toJsonStringWithUser(Detector detector) throws IOException return BytesReference.bytes(builder).utf8ToString(); } + public static String toJsonString(IocFinding iocFinding) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder = iocFinding.toXContent(builder, ToXContent.EMPTY_PARAMS); + return BytesReference.bytes(builder).utf8ToString(); + } + + public static String toJsonString(ThreatIntelAlert alert) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder = alert.toXContent(builder, ToXContent.EMPTY_PARAMS); + return BytesReference.bytes(builder).utf8ToString(); + } + public static String toJsonString(ThreatIntelFeedData threatIntelFeedData) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); builder = threatIntelFeedData.toXContent(builder, ToXContent.EMPTY_PARAMS); @@ -861,6 +900,16 @@ public static Action randomAction(String destinationId) { return new Action(name, destinationId, template, template, throttleEnabled, throttle, OpenSearchRestTestCase.randomAlphaOfLength(10), null); } + public static Action randomThreatInteMonitorAction(String destinationId) { + String name = OpenSearchRestTestCase.randomUnicodeOfLength(10); + Script template = randomTemplateScript("Threat intel Monitor {{ctx.monitor.name}} just entered alert status. Please investigate the issue.\n" + + " - Trigger: {{ctx.trigger.name}}\n" + + " - Severity: {{ctx.trigger.severity}}", null); + Boolean throttleEnabled = false; + Throttle throttle = randomThrottle(null, null); + return new Action(name, destinationId, template, template, throttleEnabled, throttle, OpenSearchRestTestCase.randomAlphaOfLength(10), null); + } + public static Script randomTemplateScript(String source, Map params) { if (params == null) { params = new HashMap<>(); @@ -943,7 +992,7 @@ public static String netFlowMappings() { " }"; } - public static String productIndexMapping(){ + public static String productIndexMapping() { return "\"properties\":{\n" + " \"name\":{\n" + " \"type\":\"keyword\"\n" + @@ -964,7 +1013,7 @@ public static String productIndexMapping(){ "}"; } - public static String productIndexAvgAggRule(){ + public static String productIndexAvgAggRule() { return " title: Test\n" + " id: 39f918f3-981b-4e6f-a975-8af7e507ef2b\n" + " status: test\n" + @@ -984,7 +1033,7 @@ public static String productIndexAvgAggRule(){ " condition: sel | avg(fieldA) by fieldC > 110"; } - public static String productIndexCountAggRule(){ + public static String productIndexCountAggRule() { return " title: Test\n" + " id: 39f918f3-981b-4e6f-a975-8af7e507ef2b\n" + " status: test\n" + @@ -1002,7 +1051,7 @@ public static String productIndexCountAggRule(){ " condition: sel | count(*) by name > 2"; } - public static String randomAggregationRule(String aggFunction, String signAndValue) { + public static String randomAggregationRule(String aggFunction, String signAndValue) { String rule = "title: Remote Encrypting File System Abuse\n" + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + @@ -1033,7 +1082,7 @@ public static String randomAggregationRule(String aggFunction, String signAndVa return String.format(Locale.ROOT, rule, aggFunction, signAndValue); } - public static String randomAggregationRule(String aggFunction, String signAndValue, String opCode) { + public static String randomAggregationRule(String aggFunction, String signAndValue, String opCode) { String rule = "title: Remote Encrypting File System Abuse\n" + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + @@ -1065,7 +1114,7 @@ public static String randomAggregationRule(String aggFunction, String signAndVa } public static String randomCloudtrailAggrRule() { - return "id: c64c5175-5189-431b-a55e-6d9882158250\n" + + return "id: c64c5175-5189-431b-a55e-6d9882158250\n" + "logsource:\n" + " product: cloudtrail\n" + "title: Accounts created and deleted within 24h\n" + @@ -1110,7 +1159,7 @@ public static String randomCloudtrailAggrRuleWithDotFields() { " - lambda.amazonaws.com\n" + " api.operation: \n" + " - Invoke\n" + - " timeframe: 20m\n" + + " timeframe: 1m\n" + " tags:\n" + " - attack.privilege_escalation\n" + " - attack.t1078"; @@ -1835,9 +1884,71 @@ public static String windowsIndexMappingOnlyNumericAndText() { " }"; } + public static String oldThreatIntelJobMapping() { + return " \"dynamic\": \"strict\",\n" + + " \"_meta\": {\n" + + " \"schema_version\": 1\n" + + " },\n" + + " \"properties\": {\n" + + " \"schema_version\": {\n" + + " \"type\": \"integer\"\n" + + " },\n" + + " \"enabled_time\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"indices\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"last_update_time\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"name\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"schedule\": {\n" + + " \"properties\": {\n" + + " \"interval\": {\n" + + " \"properties\": {\n" + + " \"period\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"start_time\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"unit\": {\n" + + " \"type\": \"text\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"state\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"update_enabled\": {\n" + + " \"type\": \"boolean\"\n" + + " },\n" + + " \"update_stats\": {\n" + + " \"properties\": {\n" + + " \"last_failed_at_in_epoch_millis\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"last_processing_time_in_millis\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"last_skipped_at_in_epoch_millis\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"last_succeeded_at_in_epoch_millis\": {\n" + + " \"type\": \"long\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }"; + } - public static String randomDoc(int severity, int version, String opCode) { - String doc = "{\n" + + public static String randomDoc(int severity, int version, String opCode) { + String doc = "{\n" + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" + "\"Keywords\":\"9223372036854775808\",\n" + @@ -1876,7 +1987,7 @@ public static String randomDoc(int severity, int version, String opCode) { } public static String randomDocForNotCondition(int severity, int version, String opCode) { - String doc = "{\n" + + String doc = "{\n" + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" + "\"Keywords\":\"9223372036854775808\",\n" + @@ -1914,7 +2025,7 @@ public static String randomDocForNotCondition(int severity, int version, String } public static String randomDocOnlyNumericAndDate(int severity, int version, String opCode) { - String doc = "{\n" + + String doc = "{\n" + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + "\"ExecutionProcessID\":2001,\n" + "\"ExecutionThreadID\":2616,\n" + @@ -1925,7 +2036,7 @@ public static String randomDocOnlyNumericAndDate(int severity, int version, Stri } public static String randomDocOnlyNumericAndText(int severity, int version, String opCode) { - String doc = "{\n" + + String doc = "{\n" + "\"TaskName\":\"SYSTEM\",\n" + "\"ExecutionProcessID\":2001,\n" + "\"ExecutionThreadID\":2616,\n" + @@ -1936,8 +2047,8 @@ public static String randomDocOnlyNumericAndText(int severity, int version, Stri } //Add IPs in HostName field. - public static String randomDocWithIpIoc(int severity, int version, String ioc) { - String doc = "{\n" + + public static String randomDocWithIpIoc(int severity, int version, String ioc) { + String doc = "{\n" + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + "\"HostName\":\"%s\",\n" + "\"Keywords\":\"9223372036854775808\",\n" + @@ -2197,7 +2308,7 @@ public static String randomCloudtrailOcsfDoc() { " },\n" + " \"status\": \"Success\",\n" + " \"status_id\": 1,\n" + - " \"time\": 1702952105000,\n" + + " \"time\": " + System.currentTimeMillis() + ",\n" + " \"type_name\": \"Account Change: Detach Policy\",\n" + " \"type_uid\": 300108,\n" + " \"unmapped\": {\n" + @@ -2705,4 +2816,178 @@ public static NamedXContentRegistry xContentRegistry() { public static XContentBuilder builder() throws IOException { return XContentBuilder.builder(XContentType.JSON.xContent()); } + + public static SATIFSourceConfigDto randomSATIFSourceConfigDto() { + return randomSATIFSourceConfigDto( + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); + } + + public static SATIFSourceConfigDto randomSATIFSourceConfigDto( + String feedName, + String feedFormat, + SourceConfigType sourceConfigType, + User createdByUser, + Instant createdAt, + Source source, + String description, + Instant enabledTime, + Instant lastUpdateTime, + org.opensearch.jobscheduler.spi.schedule.IntervalSchedule schedule, + TIFJobState state, + RefreshType refreshType, + Instant lastRefreshedTime, + User lastRefreshedUser, + Boolean isEnabled, + List iocTypes + ) { + if (feedName == null) { + feedName = randomString(); + } + if (feedFormat == null) { + feedFormat = "STIX"; + } + if (sourceConfigType == null) { + sourceConfigType = SourceConfigType.S3_CUSTOM; + } + if (isEnabled == null) { + isEnabled = true; + } + if (source == null) { + source = new S3Source("bucket", "objectkey", "region", "rolearn"); + } + if (schedule == null) { + schedule = new org.opensearch.jobscheduler.spi.schedule.IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); + } + if (iocTypes == null) { + iocTypes = List.of("ip"); + } + + return new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + description, + createdByUser, + createdAt, + source, + enabledTime, + lastUpdateTime, + schedule, + state, + refreshType, + lastRefreshedTime, + lastRefreshedUser, + isEnabled, + iocTypes, + true + ); + } + + public static SATIFSourceConfig randomSATIFSourceConfig() { + return randomSATIFSourceConfig( + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); + } + + public static SATIFSourceConfig randomSATIFSourceConfig( + String feedName, + String feedFormat, + SourceConfigType sourceConfigType, + User createdByUser, + Instant createdAt, + Source source, + String description, + Instant enabledTime, + Instant lastUpdateTime, + org.opensearch.jobscheduler.spi.schedule.IntervalSchedule schedule, + TIFJobState state, + RefreshType refreshType, + Instant lastRefreshedTime, + User lastRefreshedUser, + Boolean isEnabled, + IocStoreConfig iocStoreConfig, + List iocTypes + ) { + if (feedName == null) { + feedName = randomString(); + } + if (feedFormat == null) { + feedFormat = "STIX"; + } + if (sourceConfigType == null) { + sourceConfigType = SourceConfigType.S3_CUSTOM; + } + if (isEnabled == null) { + isEnabled = true; + } + if (source == null) { + source = new S3Source("bucket", "objectkey", "region", "rolearn"); + } + if (schedule == null) { + schedule = new org.opensearch.jobscheduler.spi.schedule.IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); + } + if (iocStoreConfig == null) { + iocStoreConfig = new DefaultIocStoreConfig(List.of(new DefaultIocStoreConfig.IocToIndexDetails(new IOCType(IOCType.DOMAIN_NAME_TYPE), "indexPattern", "writeIndex"))); + } + if (iocTypes == null) { + iocTypes = List.of("ip"); + } + + return new SATIFSourceConfig( + null, + null, + feedName, + feedFormat, + sourceConfigType, + description, + new User("wrgrer", List.of("b1"), List.of("r1"), List.of("ca")), + createdAt, + source, + enabledTime, + lastUpdateTime, + schedule, + state, + refreshType, + lastRefreshedTime, + lastRefreshedUser, + isEnabled, + iocStoreConfig, + iocTypes, + true + ); + } } diff --git a/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigActionTests.java b/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigActionTests.java new file mode 100644 index 000000000..f0b932472 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigActionTests.java @@ -0,0 +1,16 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.action; + +import org.junit.Assert; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigAction; +import org.opensearch.test.OpenSearchTestCase; + +public class GetTIFSourceConfigActionTests extends OpenSearchTestCase { + public void testGetTIFSourceConfigActionName() { + Assert.assertNotNull(SAGetTIFSourceConfigAction.INSTANCE.name()); + Assert.assertEquals(SAGetTIFSourceConfigAction.INSTANCE.name(), SAGetTIFSourceConfigAction.NAME); + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigRequestTests.java b/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigRequestTests.java new file mode 100644 index 000000000..376d10b01 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigRequestTests.java @@ -0,0 +1,44 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.action; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.common.UUIDs; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigRequest; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +public class GetTIFSourceConfigRequestTests extends OpenSearchTestCase { + public void testStreamInOut() throws IOException { + BytesStreamOutput out = new BytesStreamOutput(); + String id = UUIDs.base64UUID(); + Long version = 1L; + + SAGetTIFSourceConfigRequest request = new SAGetTIFSourceConfigRequest(id, version); + request.writeTo(out); + + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + SAGetTIFSourceConfigRequest newReq = new SAGetTIFSourceConfigRequest(sin); + + assertEquals(id, newReq.getId()); + assertEquals(version, newReq.getVersion()); + } + + public void testValidate() { + String id = UUIDs.base64UUID(); + Long version = 1L; + + SAGetTIFSourceConfigRequest request = new SAGetTIFSourceConfigRequest(id, version); + ActionRequestValidationException e = request.validate(); + assertNull(e); + + request = new SAGetTIFSourceConfigRequest("", 0L); + e = request.validate(); + assertNotNull(e); + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigResponseTests.java b/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigResponseTests.java new file mode 100644 index 000000000..9acb3da4e --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigResponseTests.java @@ -0,0 +1,88 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.action; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.Assert; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigResponse; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; +import org.opensearch.securityanalytics.threatIntel.model.S3Source; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.model.Source; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; + +public class GetTIFSourceConfigResponseTests extends OpenSearchTestCase { + private static final Logger log = LogManager.getLogger(GetTIFSourceConfigResponseTests.class); + + public void testStreamInOut() throws IOException { + String name = "test_feed_name"; + String format = "STIX"; + SourceConfigType sourceConfigType = SourceConfigType.S3_CUSTOM; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); + Source source = new S3Source("bucket", "objectkey", "region", "rolearn"); + List iocTypes = List.of("hash"); + + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + name, + format, + sourceConfigType, + null, + null, + Instant.now(), + source, + null, + Instant.now(), + schedule, + null, + null, + Instant.now(), + null, + false, + iocTypes, + true + ); + + SAGetTIFSourceConfigResponse response = new SAGetTIFSourceConfigResponse(saTifSourceConfigDto.getId(), saTifSourceConfigDto.getVersion(), RestStatus.OK, saTifSourceConfigDto); + Assert.assertNotNull(response); + + BytesStreamOutput out = new BytesStreamOutput(); + response.writeTo(out); + + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + SAGetTIFSourceConfigResponse newResponse = new SAGetTIFSourceConfigResponse(sin); + + Assert.assertEquals(saTifSourceConfigDto.getId(), newResponse.getId()); + Assert.assertEquals(saTifSourceConfigDto.getVersion(), newResponse.getVersion()); + Assert.assertEquals(RestStatus.OK, newResponse.getStatus()); + Assert.assertNotNull(newResponse.getSaTifSourceConfigDto()); + Assert.assertEquals(name, newResponse.getSaTifSourceConfigDto().getName()); + Assert.assertEquals(format, newResponse.getSaTifSourceConfigDto().getFormat()); + Assert.assertEquals(sourceConfigType, newResponse.getSaTifSourceConfigDto().getType()); + Assert.assertEquals(saTifSourceConfigDto.getState(), newResponse.getSaTifSourceConfigDto().getState()); + Assert.assertEquals(saTifSourceConfigDto.getEnabledTime(), newResponse.getSaTifSourceConfigDto().getEnabledTime()); + Assert.assertEquals(saTifSourceConfigDto.getCreatedAt(), newResponse.getSaTifSourceConfigDto().getCreatedAt()); + Assert.assertEquals(saTifSourceConfigDto.getLastUpdateTime(), newResponse.getSaTifSourceConfigDto().getLastUpdateTime()); + Assert.assertEquals(saTifSourceConfigDto.isEnabled(), newResponse.getSaTifSourceConfigDto().isEnabled()); + Assert.assertEquals(saTifSourceConfigDto.getLastRefreshedTime(), newResponse.getSaTifSourceConfigDto().getLastRefreshedTime()); + Assert.assertEquals(saTifSourceConfigDto.getLastRefreshedUser(), newResponse.getSaTifSourceConfigDto().getLastRefreshedUser()); + Assert.assertEquals(schedule, newResponse.getSaTifSourceConfigDto().getSchedule()); + Assert.assertEquals(saTifSourceConfigDto.getCreatedByUser(), newResponse.getSaTifSourceConfigDto().getCreatedByUser()); + Assert.assertTrue(iocTypes.containsAll(newResponse.getSaTifSourceConfigDto().getIocTypes()) && + newResponse.getSaTifSourceConfigDto().getIocTypes().containsAll(iocTypes)); + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigActionTests.java b/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigActionTests.java new file mode 100644 index 000000000..c8b8b29bd --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigActionTests.java @@ -0,0 +1,16 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.action; + +import org.junit.Assert; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; +import org.opensearch.test.OpenSearchTestCase; + +public class IndexTIFSourceConfigActionTests extends OpenSearchTestCase { + public void testIndexTIFSourceConfigActionName() { + Assert.assertNotNull(SAIndexTIFSourceConfigAction.INSTANCE.name()); + Assert.assertEquals(SAIndexTIFSourceConfigAction.INSTANCE.name(), SAIndexTIFSourceConfigAction.NAME); + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigRequestTests.java b/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigRequestTests.java new file mode 100644 index 000000000..e40516e25 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigRequestTests.java @@ -0,0 +1,73 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.action; + +import org.junit.Assert; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.rest.RestRequest; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigRequest; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +import static org.opensearch.securityanalytics.TestHelpers.randomSATIFSourceConfigDto; + +public class IndexTIFSourceConfigRequestTests extends OpenSearchTestCase { + + public void testTIFSourceConfigPostRequest() throws IOException { + SATIFSourceConfigDto saTifSourceConfigDto = randomSATIFSourceConfigDto(); + String id = saTifSourceConfigDto.getId(); + SAIndexTIFSourceConfigRequest request = new SAIndexTIFSourceConfigRequest(id, RestRequest.Method.POST, saTifSourceConfigDto); + Assert.assertNotNull(request); + + BytesStreamOutput out = new BytesStreamOutput(); + request.writeTo(out); + + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + SAIndexTIFSourceConfigRequest newRequest = new SAIndexTIFSourceConfigRequest(sin); + Assert.assertEquals(id, request.getTIFConfigId()); + Assert.assertEquals(RestRequest.Method.POST, newRequest.getMethod()); + Assert.assertNotNull(newRequest.getTIFConfigDto()); + } + + public void testValidateSourceConfigPostRequest() { + // Source config with invalid: name, format, source, ioc type, source config type + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + false, + null, + true + ); + String id = saTifSourceConfigDto.getId(); + SAIndexTIFSourceConfigRequest request = new SAIndexTIFSourceConfigRequest(id, RestRequest.Method.POST, saTifSourceConfigDto); + Assert.assertNotNull(request); + + ActionRequestValidationException exception = request.validate(); + assertEquals(5, exception.validationErrors().size()); + assertTrue(exception.validationErrors().contains("Name must not be empty")); + assertTrue(exception.validationErrors().contains("Format must not be empty")); + assertTrue(exception.validationErrors().contains("Source must not be empty")); + assertTrue(exception.validationErrors().contains("Must specify at least one IOC type")); + assertTrue(exception.validationErrors().contains("Type must not be empty")); + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigResponseTests.java b/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigResponseTests.java new file mode 100644 index 000000000..f720099bf --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigResponseTests.java @@ -0,0 +1,76 @@ +package org.opensearch.securityanalytics.action; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.Assert; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigResponse; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; +import org.opensearch.securityanalytics.threatIntel.model.S3Source; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.model.Source; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; + +public class IndexTIFSourceConfigResponseTests extends OpenSearchTestCase { + + private static final Logger log = LogManager.getLogger(IndexTIFSourceConfigResponseTests.class); + + public void testIndexTIFSourceConfigPostResponse() throws IOException { + String name = "feed_Name"; + String format = "STIX"; + SourceConfigType sourceConfigType = SourceConfigType.S3_CUSTOM; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); + Source source = new S3Source("bucket", "objectkey", "region", "rolearn"); + List iocTypes = List.of("hash"); + + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + name, + format, + sourceConfigType, + null, + null, + Instant.now(), + source, + null, + Instant.now(), + schedule, + null, + null, + Instant.now(), + null, + false, + iocTypes, + true + ); + + SAIndexTIFSourceConfigResponse response = new SAIndexTIFSourceConfigResponse(saTifSourceConfigDto.getId(), saTifSourceConfigDto.getVersion(), RestStatus.OK, saTifSourceConfigDto); + Assert.assertNotNull(response); + + BytesStreamOutput out = new BytesStreamOutput(); + response.writeTo(out); + + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + SAIndexTIFSourceConfigResponse newResponse = new SAIndexTIFSourceConfigResponse(sin); + + Assert.assertEquals(saTifSourceConfigDto.getId(), newResponse.getTIFConfigId()); + Assert.assertEquals(saTifSourceConfigDto.getVersion(), newResponse.getVersion()); + Assert.assertEquals(RestStatus.OK, newResponse.getStatus()); + Assert.assertNotNull(newResponse.getTIFConfigDto()); + Assert.assertEquals(name, newResponse.getTIFConfigDto().getName()); + Assert.assertEquals(format, newResponse.getTIFConfigDto().getFormat()); + Assert.assertEquals(sourceConfigType, newResponse.getTIFConfigDto().getType()); + Assert.assertEquals(schedule, newResponse.getTIFConfigDto().getSchedule()); + Assert.assertTrue(iocTypes.containsAll(newResponse.getTIFConfigDto().getIocTypes()) && + newResponse.getTIFConfigDto().getIocTypes().containsAll(iocTypes)); + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java index 3542ee309..82d6ecc5c 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java @@ -88,13 +88,15 @@ public void testGetAlerts_success() { new CronSchedule("31 * * * *", ZoneId.of("Asia/Kolkata"), Instant.ofEpochSecond(1538164858L)), Instant.now(), Instant.now(), - Monitor.MonitorType.DOC_LEVEL_MONITOR, + Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), null, 1, List.of(), List.of(), Map.of(), new DataSources(), + true, + null, TransportIndexDetectorAction.PLUGIN_OWNER_FIELD ), new DocumentLevelTrigger("trigger_id_1", "my_trigger", "severity_low", List.of(), new Script("")), @@ -122,13 +124,15 @@ public void testGetAlerts_success() { new CronSchedule("31 * * * *", ZoneId.of("Asia/Kolkata"), Instant.ofEpochSecond(1538164858L)), Instant.now(), Instant.now(), - Monitor.MonitorType.DOC_LEVEL_MONITOR, + Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), null, 1, List.of(), List.of(), Map.of(), new DataSources(), + true, + null, TransportIndexDetectorAction.PLUGIN_OWNER_FIELD ), new DocumentLevelTrigger("trigger_id_1", "my_trigger", "severity_low", List.of(), new Script("")), @@ -191,10 +195,10 @@ public void testGetAlerts_success() { ); doAnswer(invocation -> { - ActionListener l = invocation.getArgument(6); + ActionListener l = invocation.getArgument(8); l.onResponse(getAlertsResponse); return null; - }).when(alertssService).getAlertsByMonitorIds(any(), any(), anyString(), any(Table.class), anyString(), anyString(), any(ActionListener.class)); + }).when(alertssService).getAlertsByMonitorIds(any(), any(), anyString(), any(Table.class), anyString(), anyString(), any(), any(), any(ActionListener.class)); // Call getFindingsByDetectorId Table table = new Table( @@ -205,7 +209,8 @@ public void testGetAlerts_success() { 0, null ); - alertssService.getAlertsByDetectorId("detector_id123", table, "severity_low", Alert.State.COMPLETED.toString(), new ActionListener<>() { + alertssService.getAlertsByDetectorId("detector_id123", table, "severity_low", Alert.State.COMPLETED.toString(), null, null, + new ActionListener<>() { @Override public void onResponse(GetAlertsResponse getAlertsResponse) { assertEquals(2, (int)getAlertsResponse.getTotalAlerts()); @@ -258,10 +263,10 @@ public void testGetFindings_getFindingsByMonitorIdFailures() { }).when(client).execute(eq(GetDetectorAction.INSTANCE), any(GetDetectorRequest.class), any(ActionListener.class)); doAnswer(invocation -> { - ActionListener l = invocation.getArgument(6); + ActionListener l = invocation.getArgument(8); l.onFailure(new IllegalArgumentException("Error getting findings")); return null; - }).when(alertssService).getAlertsByMonitorIds(any(), any(), anyString(), any(Table.class), anyString(), anyString(), any(ActionListener.class)); + }).when(alertssService).getAlertsByMonitorIds(any(), any(), anyString(), any(Table.class), anyString(), anyString(), any(), any(), any(ActionListener.class)); // Call getFindingsByDetectorId Table table = new Table( @@ -272,7 +277,8 @@ public void testGetFindings_getFindingsByMonitorIdFailures() { 0, null ); - alertssService.getAlertsByDetectorId("detector_id123", table, "severity_low", Alert.State.COMPLETED.toString(), new ActionListener<>() { + alertssService.getAlertsByDetectorId("detector_id123", table, "severity_low", Alert.State.COMPLETED.toString(), null, null, + new ActionListener<>() { @Override public void onResponse(GetAlertsResponse getAlertsResponse) { fail("this test should've failed"); @@ -307,7 +313,8 @@ public void testGetFindings_getDetectorFailure() { 0, null ); - alertssService.getAlertsByDetectorId("detector_id123", table, "severity_low", Alert.State.COMPLETED.toString(), new ActionListener<>() { + alertssService.getAlertsByDetectorId("detector_id123", table, "severity_low", Alert.State.COMPLETED.toString(), null, null, + new ActionListener<>() { @Override public void onResponse(GetAlertsResponse getAlertsResponse) { fail("this test should've failed"); diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java index c666a1d27..982a28168 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java @@ -6,16 +6,17 @@ package org.opensearch.securityanalytics.alerts; import java.io.IOException; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import org.apache.http.HttpStatus; import org.apache.http.entity.StringEntity; @@ -26,7 +27,6 @@ import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; -import org.opensearch.commons.alerting.model.Monitor; import org.opensearch.commons.alerting.model.action.Action; import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; @@ -38,19 +38,15 @@ import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; +import org.opensearch.test.rest.OpenSearchRestTestCase; -import static java.util.Collections.emptyList; import static org.opensearch.securityanalytics.TestHelpers.netFlowMappings; import static org.opensearch.securityanalytics.TestHelpers.randomAction; import static org.opensearch.securityanalytics.TestHelpers.randomAggregationRule; -import static org.opensearch.securityanalytics.TestHelpers.randomDetector; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; -import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputs; -import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndThreatIntel; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndTriggers; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithTriggers; import static org.opensearch.securityanalytics.TestHelpers.randomDoc; -import static org.opensearch.securityanalytics.TestHelpers.randomDocWithIpIoc; import static org.opensearch.securityanalytics.TestHelpers.randomNetworkDoc; import static org.opensearch.securityanalytics.TestHelpers.randomIndex; import static org.opensearch.securityanalytics.TestHelpers.randomRule; @@ -59,7 +55,6 @@ import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_RETENTION_PERIOD; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_ROLLOVER_PERIOD; -import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE; public class AlertsIT extends SecurityAnalyticsRestTestCase { @@ -180,8 +175,124 @@ public void testGetAlerts_success() throws IOException { assertEquals(((ArrayList) ackAlertsResponseMap.get("acknowledged")).size(), 1); } + @Ignore + @SuppressWarnings("unchecked") + public void testGetAlertsByStartTimeAndEndTimeSuccess() throws IOException, InterruptedException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + String rule = randomRule(); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", randomDetectorType()), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String createdId = responseBody.get("_id").toString(); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + createAlertingMonitorConfigIndex(null); + Action triggerAction = randomAction(createDestination()); + + Detector detector = randomDetectorWithInputsAndTriggers(List.of(new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(createdId)), + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()))), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(createdId), List.of(), List.of("attack.defense_evasion"), List.of(triggerAction), List.of()))); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + final String detectorId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", randomDoc()); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(6, noOfSigmaRuleMatches); + + Assert.assertEquals(1, ((Map) executeResults.get("trigger_results")).values().size()); + + // Call GetAlerts API + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.ALERTS_BASE_URI, params, null); + Map getAlertsBody = asMap(getAlertsResponse); + // TODO enable asserts here when able + Assert.assertEquals(1, getAlertsBody.get("total_alerts")); + + Instant startTime = Instant.now(); + indexDoc(index, "2", randomDoc()); + indexDoc(index, "5", randomDoc()); + + executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(6, noOfSigmaRuleMatches); + + Assert.assertEquals(1, ((Map) executeResults.get("trigger_results")).values().size()); + Instant endTime = Instant.now(); + + indexDoc(index, "4", randomDoc()); + indexDoc(index, "6", randomDoc()); + + executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(6, noOfSigmaRuleMatches); + + AtomicBoolean success = new AtomicBoolean(true); + OpenSearchRestTestCase.waitUntil( + () -> { + try { + // Call GetAlerts API + Map alertParams = new HashMap<>(); + alertParams.put("detector_id", detectorId); + alertParams.put("startTime", String.valueOf(startTime.toEpochMilli())); + alertParams.put("endTime", String.valueOf(endTime.toEpochMilli())); + Response currGetAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.ALERTS_BASE_URI, alertParams, null); + Map currGetAlertsBody = asMap(currGetAlertsResponse); + // TODO enable asserts here when able + success.set(Integer.parseInt(currGetAlertsBody.get("total_alerts").toString()) == 2); + } catch (IOException ex) { + success.set(false); + } + return success.get(); + }, 2, TimeUnit.MINUTES + ); + Assert.assertTrue(success.get()); + } + public void testGetAlerts_noDetector_failure() throws IOException { - // Call GetAlerts API + // Call GetAlerts API Map params = new HashMap<>(); params.put("detector_id", "nonexistent_detector_id"); try { @@ -243,7 +354,7 @@ public void testAckAlerts_WithInvalidDetectorAlertsCombination() throws IOExcept indexDoc(index, "1", randomDoc()); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); Response executeResponse = null; @@ -451,7 +562,7 @@ public void testGetAlerts_byDetectorType_success() throws IOException, Interrupt indexDoc(index, "1", randomDoc()); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); Map executeResults = entityAsMap(executeResponse); @@ -567,7 +678,7 @@ public void testGetAlerts_byDetectorType_multipleDetectors_success() throws IOEx noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); Assert.assertEquals(1, noOfSigmaRuleMatches); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); request = "{\n" + " \"query\" : {\n" + @@ -585,7 +696,7 @@ public void testGetAlerts_byDetectorType_multipleDetectors_success() throws IOEx hits = executeSearch(DetectorMonitorConfig.getAlertsIndex("network"), request); } - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Call GetAlerts API for WINDOWS detector Map params = new HashMap<>(); @@ -680,7 +791,7 @@ public void testAlertHistoryRollover_maxAge() throws IOException, InterruptedExc * * @throws IOException */ - public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException { + public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException, InterruptedException { String index = createTestIndex(randomIndex(), windowsIndexMapping()); Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); @@ -693,7 +804,7 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException Response createMappingResponse = client().performRequest(createMappingRequest); - assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + assertEquals(org.apache.http.HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); String infoOpCode = "Info"; @@ -706,7 +817,7 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException Collections.emptyList()); Detector detector = randomDetectorWithInputsAndTriggers(List.of(input), List.of(new DetectorTrigger("randomtrigegr", "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of())) - ); + ); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); @@ -717,7 +828,7 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(1, response.getHits().getTotalHits().value); // 5 for rules, 1 for match_all query in chained findings monitor @@ -735,28 +846,11 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException SearchHit hit = hits.get(0); Map updatedDetectorMap = (HashMap) (hit.getSourceAsMap().get("detector")); - List monitorIds = ((List) (updatedDetectorMap).get("monitor_id")); + String workflowId = ((List) (updatedDetectorMap).get("workflow_ids")).get(0); indexDoc(index, "1", randomDoc(2, 4, infoOpCode)); indexDoc(index, "2", randomDoc(3, 4, infoOpCode)); - - Map numberOfMonitorTypes = new HashMap<>(); - - for (String monitorId : monitorIds) { - Map monitor = (Map) (entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId)))).get("monitor"); - numberOfMonitorTypes.merge(monitor.get("monitor_type"), 1, Integer::sum); - Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); - - // Assert monitor executions - Map executeResults = entityAsMap(executeResponse); - if (Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue().equals(monitor.get("monitor_type")) && false == monitor.get("name").equals(detector.getName() + "_chained_findings")) { - int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); - assertEquals(5, noOfSigmaRuleMatches); - } - } - - assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.BUCKET_LEVEL_MONITOR.getValue()).intValue()); - assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue()).intValue()); + executeAlertingWorkflow(workflowId, Collections.emptyMap()); Map params = new HashMap<>(); params.put("detector_id", detectorId); @@ -780,13 +874,13 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException List> queries = (List>) finding.get("queries"); Set findingRuleIds = queries.stream().map(it -> it.get("id").toString()).collect(Collectors.toSet()); - // In the case of bucket level monitors, queries will always contain one value - String aggRuleId = findingRuleIds.iterator().next(); - List findingDocs = (List) finding.get("related_doc_ids"); + // In the case of bucket level monitors, queries will always contain one value + String aggRuleId = findingRuleIds.iterator().next(); + List findingDocs = (List) finding.get("related_doc_ids"); - if (aggRuleId.equals(sumRuleId)) { - assertTrue(List.of("1", "2", "3", "4", "5", "6", "7").containsAll(findingDocs)); - } + if (aggRuleId.equals(sumRuleId)) { + assertTrue(List.of("1", "2", "3", "4", "5", "6", "7").containsAll(findingDocs)); + } } assertTrue(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8").containsAll(docLevelFinding)); @@ -796,7 +890,7 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.ALERTS_BASE_URI, params1, null); Map getAlertsBody = asMap(getAlertsResponse); // TODO enable asserts here when able - Assert.assertEquals(3, getAlertsBody.get("total_alerts")); // 2 doc level alerts for each doc, 1 bucket level alert + Assert.assertEquals(1, getAlertsBody.get("total_alerts")); // 2 doc level alerts for each doc, 1 bucket level alert input = new DetectorInput("updated", List.of("windows"), detectorRules, Collections.emptyList()); @@ -804,7 +898,7 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException List.of(new DetectorTrigger("updated", "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of())) ); /** update detector and verify chained findings monitor should still exist*/ - Response updateResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId, Collections.emptyMap(), toHttpEntity(updatedDetector)); + makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId, Collections.emptyMap(), toHttpEntity(updatedDetector)); hits = executeSearch(Detector.DETECTORS_INDEX, request); hit = hits.get(0); updatedDetectorMap = (HashMap) (hit.getSourceAsMap().get("detector")); @@ -817,29 +911,191 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException hit = hits.get(0); updatedDetectorMap = (HashMap) (hit.getSourceAsMap().get("detector")); - monitorIds = ((List) (updatedDetectorMap).get("monitor_id")); - numberOfMonitorTypes = new HashMap<>(); - for (String monitorId : monitorIds) { - Map monitor = (Map) (entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId)))).get("monitor"); - numberOfMonitorTypes.merge(monitor.get("monitor_type"), 1, Integer::sum); - Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); - - // Assert monitor executions - Map executeResults = entityAsMap(executeResponse); - - if (Monitor.MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(monitor.get("monitor_type"))) { - ArrayList triggerResults = new ArrayList(((Map) executeResults.get("trigger_results")).values()); - assertEquals(triggerResults.size(), 1); - Map triggerResult = (Map) triggerResults.get(0); - assertTrue(triggerResult.containsKey("agg_result_buckets")); - HashMap aggResultBuckets = (HashMap) triggerResult.get("agg_result_buckets"); - assertTrue(aggResultBuckets.containsKey("4")); - assertTrue(aggResultBuckets.containsKey("5")); + workflowId = ((List) (updatedDetectorMap).get("workflow_ids")).get(0); + executeAlertingWorkflow(workflowId, Collections.emptyMap()); + + params = new HashMap<>(); + params.put("detector_id", detectorId); + getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + getFindingsBody = entityAsMap(getFindingsResponse); + + assertNotNull(getFindingsBody); + assertEquals(2, getFindingsBody.get("total_findings")); + + findingDetectorId = ((Map) ((List) getFindingsBody.get("findings")).get(0)).get("detectorId").toString(); + assertEquals(detectorId, findingDetectorId); + + findingIndex = ((Map) ((List) getFindingsBody.get("findings")).get(0)).get("index").toString(); + assertEquals(index, findingIndex); + + docLevelFinding = new ArrayList<>(); + findings = (List) getFindingsBody.get("findings"); + + + for (Map finding : findings) { + List> queries = (List>) finding.get("queries"); + Set findingRuleIds = queries.stream().map(it -> it.get("id").toString()).collect(Collectors.toSet()); + + // In the case of bucket level monitors, queries will always contain one value + String aggRuleId = findingRuleIds.iterator().next(); + List findingDocs = (List) finding.get("related_doc_ids"); + + if (aggRuleId.equals(sumRuleId)) { + assertTrue(List.of("1", "2", "3", "4", "5", "6", "7").containsAll(findingDocs)); } } - assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.BUCKET_LEVEL_MONITOR.getValue()).intValue()); - assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue()).intValue()); + AtomicBoolean alertRespStatus = new AtomicBoolean(false); + OpenSearchRestTestCase.waitUntil( + () -> { + Map queryParams = new HashMap<>(); + queryParams.put("detector_id", detectorId); + try { + Response alertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.ALERTS_BASE_URI, queryParams, null); + Map alertsBody = asMap(alertsResponse); + // TODO enable asserts here when able + if (Integer.parseInt(alertsBody.get("total_alerts").toString()) == 2) { + alertRespStatus.set(true); + return true; + } + return false; + } catch (IOException e) { + return false; + } + }, 2, TimeUnit.MINUTES); + Assert.assertTrue(alertRespStatus.get()); + } + + public void test_detectorWith1AggRuleAndTriggeronRule_updateWithSecondAggRule() throws IOException, InterruptedException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + + assertEquals(org.apache.http.HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + + String infoOpCode = "Info"; + /** 1st agg rule*/ + String sumRuleId = createRule(randomAggregationRule("sum", " > 1", infoOpCode)); + + + List detectorRules = List.of(new DetectorRule(sumRuleId)); + + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), detectorRules, + Collections.emptyList()); + Detector detector = randomDetectorWithInputsAndTriggers(List.of(input), + List.of(new DetectorTrigger("randomtrigegr", "test-trigger", "1", List.of(randomDetectorType()), List.of(sumRuleId), List.of(), List.of(), List.of(), List.of())) + ); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + assertEquals(1, response.getHits().getTotalHits().value); // 5 for rules, 1 for match_all query in chained findings monitor + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + String detectorId = responseBody.get("_id").toString(); + request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + Map updatedDetectorMap = (HashMap) (hit.getSourceAsMap().get("detector")); + + String workflowId = ((List) (updatedDetectorMap).get("workflow_ids")).get(0); + + indexDoc(index, "1", randomDoc(2, 4, infoOpCode)); + indexDoc(index, "2", randomDoc(3, 4, infoOpCode)); + executeAlertingWorkflow(workflowId, Collections.emptyMap()); + + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + /** assert findings */ + assertNotNull(getFindingsBody); + assertEquals(1, getFindingsBody.get("total_findings")); + + /**assert alerts */ + Map params1 = new HashMap<>(); + params1.put("detector_id", detectorId); + Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.ALERTS_BASE_URI, params1, null); + Map getAlertsBody = asMap(getAlertsResponse); + + Assert.assertEquals(1, getAlertsBody.get("total_alerts")); + /** 2nd agg rule*/ + String sumRuleId2 = createRule(randomAggregationRule("sum", " > 1", infoOpCode)); + String sumRuleId3 = createRule(randomAggregationRule("sum", " > 100", infoOpCode)); + + detectorRules = List.of(new DetectorRule(sumRuleId), new DetectorRule(sumRuleId2)); + input = new DetectorInput("updated", List.of("windows"), detectorRules, Collections.emptyList()); + Detector updatedDetector = randomDetectorWithInputsAndTriggers(List.of(input), + List.of(new DetectorTrigger("updated1", "test-trigger1", "1", List.of(randomDetectorType()), List.of(sumRuleId2, sumRuleId), List.of(), List.of(), List.of(), List.of()), + new DetectorTrigger("updated2", "test-trigger2", "1", List.of(randomDetectorType()), List.of(sumRuleId2, sumRuleId3), List.of(), List.of(), List.of(), List.of()), + new DetectorTrigger("noAlertsExpected", "test-trigger2", "1", List.of(randomDetectorType()), List.of(sumRuleId3), List.of(), List.of(), List.of(), List.of())) + ); + /** update detector and verify chained findings monitor should still exist*/ + makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId, Collections.emptyMap(), toHttpEntity(updatedDetector)); + hits = executeSearch(Detector.DETECTORS_INDEX, request); + hit = hits.get(0); + updatedDetectorMap = (HashMap) (hit.getSourceAsMap().get("detector")); + + assertEquals(3, ((List) (updatedDetectorMap).get("monitor_id")).size()); + indexDoc(index, "3", randomDoc(2, 5, infoOpCode)); + indexDoc(index, "4", randomDoc(3, 5, infoOpCode)); + + hits = executeSearch(Detector.DETECTORS_INDEX, request); + hit = hits.get(0); + updatedDetectorMap = (HashMap) (hit.getSourceAsMap().get("detector")); + + workflowId = ((List) (updatedDetectorMap).get("workflow_ids")).get(0); + executeAlertingWorkflow(workflowId, Collections.emptyMap()); + + params = new HashMap<>(); + params.put("detector_id", detectorId); + getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + getFindingsBody = entityAsMap(getFindingsResponse); + + assertNotNull(getFindingsBody); + assertEquals(3, getFindingsBody.get("total_findings")); + + AtomicBoolean alertsCondSatisfy = new AtomicBoolean(false); + OpenSearchRestTestCase.waitUntil( + () -> { + try { + Map queryParams = new HashMap<>(); + queryParams.put("detector_id", detectorId); + Response alertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.ALERTS_BASE_URI, queryParams, null); + Map alertsBody = asMap(alertsResponse); + if (Integer.parseInt(alertsBody.get("total_alerts").toString()) == 3) { + alertsCondSatisfy.set(true); + } + return 3 == Integer.parseInt(alertsBody.get("total_alerts").toString()); + } catch (Exception e) { + return false; + } + }, 2, TimeUnit.MINUTES + ); + Assert.assertTrue(alertsCondSatisfy.get()); } @Ignore @@ -966,7 +1222,7 @@ public void testAlertHistoryRollover_maxDocs() throws IOException, InterruptedEx indexDoc(index, "1", randomDoc()); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); Map executeResults = entityAsMap(executeResponse); diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/SecureAlertsRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/SecureAlertsRestApiIT.java index 28172037c..07e576f84 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/SecureAlertsRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/SecureAlertsRestApiIT.java @@ -257,7 +257,7 @@ public void testGetAlerts_byDetectorType_success() throws IOException, Interrupt indexDoc(index, "1", randomDoc()); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); Map executeResults = entityAsMap(executeResponse); diff --git a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java index f05092d01..b2d56c1f5 100644 --- a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java @@ -15,8 +15,6 @@ import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; import org.opensearch.securityanalytics.TestHelpers; -import org.opensearch.securityanalytics.model.CorrelationQuery; -import org.opensearch.securityanalytics.model.CorrelationRule; import org.opensearch.securityanalytics.model.CustomLogType; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; @@ -32,7 +30,6 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.function.BooleanSupplier; -import java.util.stream.Collectors; import static org.opensearch.securityanalytics.TestHelpers.*; @@ -954,304 +951,4 @@ public void testBasicCorrelationEngineWorkflowWithCustomLogTypes() throws IOExce ); } - private LogIndices createIndices() throws IOException { - LogIndices indices = new LogIndices(); - indices.adLdapLogsIndex = createTestIndex("ad_logs", adLdapLogMappings()); - indices.s3AccessLogsIndex = createTestIndex("s3_access_logs", s3AccessLogMappings()); - indices.appLogsIndex = createTestIndex("app_logs", appLogMappings()); - indices.windowsIndex = createTestIndex(randomIndex(), windowsIndexMapping()); - indices.vpcFlowsIndex = createTestIndex("vpc_flow", vpcFlowMappings()); - return indices; - } - - private String createNetworkToWindowsFieldBasedRule(LogIndices indices) throws IOException { - CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, null, "network", "srcaddr"); - CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, null, "test_windows", "SourceIp"); - - CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to windows", List.of(query1, query4), 300000L, null); - Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); - request.setJsonEntity(toJsonString(rule)); - Response response = client().performRequest(request); - - Assert.assertEquals(201, response.getStatusLine().getStatusCode()); - return entityAsMap(response).get("_id").toString(); - } - - private String createNetworkToWindowsFilterQueryBasedRule(LogIndices indices) throws IOException { - CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, "srcaddr:1.2.3.4", "network", null); - CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, "SourceIp:1.2.3.4", "test_windows", null); - - CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to windows", List.of(query1, query4), 300000L, null); - Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); - request.setJsonEntity(toJsonString(rule)); - Response response = client().performRequest(request); - - Assert.assertEquals(201, response.getStatusLine().getStatusCode()); - return entityAsMap(response).get("_id").toString(); - } - - private String createNetworkToCustomLogTypeFieldBasedRule(LogIndices indices, String customLogTypeName, String customLogTypeIndex) throws IOException { - CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, null, "network", "srcaddr"); - CorrelationQuery query4 = new CorrelationQuery(customLogTypeIndex, null, customLogTypeName, "SourceIp"); - - CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to custom log type", List.of(query1, query4), 300000L, null); - Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); - request.setJsonEntity(toJsonString(rule)); - Response response = client().performRequest(request); - - Assert.assertEquals(201, response.getStatusLine().getStatusCode()); - return entityAsMap(response).get("_id").toString(); - } - - private String createNetworkToAdLdapToWindowsRule(LogIndices indices) throws IOException { - CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, "dstaddr:4.5.6.7", "network", null); - CorrelationQuery query2 = new CorrelationQuery(indices.adLdapLogsIndex, "ResultType:50126", "ad_ldap", null); - CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, "Domain:NTAUTHORI*", "test_windows", null); - - CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to ad_ldap to windows", List.of(query1, query2, query4), 300000L, null); - Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); - request.setJsonEntity(toJsonString(rule)); - Response response = client().performRequest(request); - - Assert.assertEquals(201, response.getStatusLine().getStatusCode()); - return entityAsMap(response).get("_id").toString(); - } - - private String createWindowsToAppLogsToS3LogsRule(LogIndices indices) throws IOException { - CorrelationQuery query1 = new CorrelationQuery(indices.windowsIndex, "HostName:EC2AMAZ*", "test_windows", null); - CorrelationQuery query2 = new CorrelationQuery(indices.appLogsIndex, "endpoint:\\/customer_records.txt", "others_application", null); - CorrelationQuery query4 = new CorrelationQuery(indices.s3AccessLogsIndex, "aws.cloudtrail.eventName:ReplicateObject", "s3", null); - - CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "windows to app_logs to s3 logs", List.of(query1, query2, query4), 300000L, null); - Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); - request.setJsonEntity(toJsonString(rule)); - Response response = client().performRequest(request); - - Assert.assertEquals(201, response.getStatusLine().getStatusCode()); - return entityAsMap(response).get("_id").toString(); - } - - private String createCloudtrailFieldBasedRule(String index, String field, Long timeWindow) throws IOException { - CorrelationQuery query1 = new CorrelationQuery(index, "EventName:CreateUser", "cloudtrail", field); - CorrelationQuery query2 = new CorrelationQuery(index, "EventName:DeleteUser", "cloudtrail", field); - - CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "cloudtrail field based", List.of(query1, query2), timeWindow, null); - Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); - request.setJsonEntity(toJsonString(rule)); - Response response = client().performRequest(request); - - Assert.assertEquals(201, response.getStatusLine().getStatusCode()); - return entityAsMap(response).get("_id").toString(); - } - - @SuppressWarnings("unchecked") - private String createVpcFlowDetector(String indexName) throws IOException { - Detector vpcFlowDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("vpc flow detector for security analytics", List.of(indexName), List.of(), - getPrePackagedRules("network").stream().map(DetectorRule::new).collect(Collectors.toList()))), - List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("network"), List.of(), List.of(), List.of(), List.of(), List.of())), "network"); - - Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(vpcFlowDetector)); - Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); - - Map responseBody = asMap(createResponse); - - String createdId = responseBody.get("_id").toString(); - - String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + createdId + "\"\n" + - " }\n" + - " }\n" + - "}"; - List hits = executeSearch(Detector.DETECTORS_INDEX, request); - SearchHit hit = hits.get(0); - - return ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); - } - - @SuppressWarnings("unchecked") - private String createAdLdapDetector(String indexName) throws IOException { - // Execute CreateMappingsAction to add alias mapping for index - Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); - // both req params and req body are supported - createMappingRequest.setJsonEntity( - "{\n" + - " \"index_name\": \"" + indexName + "\",\n" + - " \"rule_topic\": \"ad_ldap\",\n" + - " \"partial\": true,\n" + - " \"alias_mappings\": {\n" + - " \"properties\": {\n" + - " \"azure.signinlogs.properties.user_id\": {\n" + - " \"path\": \"azure.signinlogs.props.user_id\",\n" + - " \"type\": \"alias\"\n" + - " },\n" + - " \"azure-platformlogs-result_type\": {\n" + - " \"path\": \"azure.platformlogs.result_type\",\n" + - " \"type\": \"alias\"\n" + - " },\n" + - " \"azure-signinlogs-result_description\": {\n" + - " \"path\": \"azure.signinlogs.result_description\",\n" + - " \"type\": \"alias\"\n" + - " },\n" + - " \"timestamp\": {\n" + - " \"path\": \"creationTime\",\n" + - " \"type\": \"alias\"\n" + - " }\n" + - " }\n" + - " }\n" + - "}" - ); - - Response response = client().performRequest(createMappingRequest); - assertEquals(RestStatus.OK.getStatus(), response.getStatusLine().getStatusCode()); - - Detector adLdapDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("ad_ldap logs detector for security analytics", List.of(indexName), List.of(), - getPrePackagedRules("ad_ldap").stream().map(DetectorRule::new).collect(Collectors.toList()))), - List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("ad_ldap"), List.of(), List.of(), List.of(), List.of(), List.of())), "ad_ldap"); - - Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(adLdapDetector)); - Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); - - Map responseBody = asMap(createResponse); - - String createdId = responseBody.get("_id").toString(); - - String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + createdId + "\"\n" + - " }\n" + - " }\n" + - "}"; - List hits = executeSearch(Detector.DETECTORS_INDEX, request); - SearchHit hit = hits.get(0); - - return ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); - } - - @SuppressWarnings("unchecked") - private String createTestWindowsDetector(String indexName) throws IOException { - // Execute CreateMappingsAction to add alias mapping for index - Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); - // both req params and req body are supported - createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + indexName + "\"," + - " \"rule_topic\":\"" + randomDetectorType() + "\", " + - " \"partial\":true" + - "}" - ); - - Response response = client().performRequest(createMappingRequest); - assertEquals(RestStatus.OK.getStatus(), response.getStatusLine().getStatusCode()); - - Detector windowsDetector = randomDetectorWithInputsAndTriggers(List.of(new DetectorInput("windows detector for security analytics", List.of(indexName), List.of(), - getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()))), - List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); - - Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(windowsDetector)); - Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); - - Map responseBody = asMap(createResponse); - - String createdId = responseBody.get("_id").toString(); - - String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + createdId + "\"\n" + - " }\n" + - " }\n" + - "}"; - List hits = executeSearch(Detector.DETECTORS_INDEX, request); - SearchHit hit = hits.get(0); - - return ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); - } - - @SuppressWarnings("unchecked") - private String createAppLogsDetector(String indexName) throws IOException { - Detector appLogsDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("app logs detector for security analytics", List.of(indexName), List.of(), - getPrePackagedRules("others_application").stream().map(DetectorRule::new).collect(Collectors.toList()))), - List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("others_application"), List.of(), List.of(), List.of(), List.of(), List.of())), "others_application"); - - Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(appLogsDetector)); - Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); - - Map responseBody = asMap(createResponse); - - String createdId = responseBody.get("_id").toString(); - - String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + createdId + "\"\n" + - " }\n" + - " }\n" + - "}"; - List hits = executeSearch(Detector.DETECTORS_INDEX, request); - SearchHit hit = hits.get(0); - - return ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); - } - - @SuppressWarnings("unchecked") - private String createS3Detector(String indexName) throws IOException { - // Execute CreateMappingsAction to add alias mapping for index - Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); - // both req params and req body are supported - createMappingRequest.setJsonEntity( - "{\n" + - " \"index_name\": \"s3_access_logs\",\n" + - " \"rule_topic\": \"s3\",\n" + - " \"partial\": true,\n" + - " \"alias_mappings\": {\n" + - " \"properties\": {\n" + - " \"aws-cloudtrail-event_source\": {\n" + - " \"type\": \"alias\",\n" + - " \"path\": \"aws.cloudtrail.event_source\"\n" + - " },\n" + - " \"aws.cloudtrail.event_name\": {\n" + - " \"type\": \"alias\",\n" + - " \"path\": \"aws.cloudtrail.event_name\"\n" + - " }\n" + - " }\n" + - " }\n" + - "}" - ); - - Response response = client().performRequest(createMappingRequest); - assertEquals(RestStatus.OK.getStatus(), response.getStatusLine().getStatusCode()); - - Detector s3AccessLogsDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("s3 access logs detector for security analytics", List.of(indexName), List.of(), - getPrePackagedRules("s3").stream().map(DetectorRule::new).collect(Collectors.toList()))), - List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("s3"), List.of(), List.of(), List.of(), List.of(), List.of())), "s3"); - - Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(s3AccessLogsDetector)); - Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); - - Map responseBody = asMap(createResponse); - - String createdId = responseBody.get("_id").toString(); - - String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + createdId + "\"\n" + - " }\n" + - " }\n" + - "}"; - List hits = executeSearch(Detector.DETECTORS_INDEX, request); - SearchHit hit = hits.get(0); - - return ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); - } - - static class LogIndices { - String vpcFlowsIndex; - String adLdapLogsIndex; - String windowsIndex; - String appLogsIndex; - String s3AccessLogsIndex; - } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRuleRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRuleRestApiIT.java index d8cdcfdc5..4694fe523 100644 --- a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRuleRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRuleRestApiIT.java @@ -15,9 +15,11 @@ import java.io.IOException; import java.util.Collections; +import java.util.List; import java.util.Map; import static org.opensearch.securityanalytics.TestHelpers.randomCorrelationRule; +import static org.opensearch.securityanalytics.TestHelpers.randomCorrelationRuleWithTrigger; public class CorrelationEngineRuleRestApiIT extends SecurityAnalyticsRestTestCase { @@ -113,4 +115,42 @@ public void testSearchCorrelationRule() throws IOException { responseMap = responseAsMap(response); Assert.assertEquals(1, Integer.parseInt(((Map) ((Map) responseMap.get("hits")).get("total")).get("value").toString())); } + + public void testSearchCorrelationRuleWithTrigger() throws IOException { + CorrelationRule rule = randomCorrelationRuleWithTrigger("custom-rule"); + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CORRELATION_RULES_BASE_URI, Collections.emptyMap(), toHttpEntity(rule)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseMap = responseAsMap(response); + Assert.assertEquals("custom-rule", ((Map) responseMap.get("rule")).get("name")); + + String request = "{\n" + + " \"query\": {\n" + + " \"nested\": {\n" + + " \"path\": \"correlate\",\n" + + " \"query\": {\n" + + " \"bool\": {\n" + + " \"must\": [\n" + + " { \"match\": {\"correlate.category\": \"network\"}}\n" + + " ]\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CORRELATION_RULES_BASE_URI + "/_search", Collections.emptyMap(), new StringEntity(request), new BasicHeader("Content-type", "application/json")); + responseMap = responseAsMap(response); + // Assuming the hits contain the matched documents + Map hits = (Map) responseMap.get("hits"); + Assert.assertNotNull(hits); + + List> hitsList = (List>) hits.get("hits"); + Assert.assertEquals(1, hitsList.size()); // Assuming you expect exactly one hit + + Map hit = hitsList.get(0); + Map source = (Map) hit.get("_source"); + Assert.assertNotNull(source); + + Object trigger = source.get("trigger"); + Assert.assertNotNull(trigger); + } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/correlation/alerts/CorrelationAlertServiceTests.java b/src/test/java/org/opensearch/securityanalytics/correlation/alerts/CorrelationAlertServiceTests.java new file mode 100644 index 000000000..6a8ea14b3 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/correlation/alerts/CorrelationAlertServiceTests.java @@ -0,0 +1,79 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.correlation.alerts; + +import org.opensearch.client.Client; +import org.opensearch.commons.alerting.model.Alert; +import org.opensearch.commons.alerting.model.CorrelationAlert; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.securityanalytics.correlation.alert.CorrelationAlertService; +import org.opensearch.securityanalytics.correlation.alert.CorrelationAlertsList; +import org.opensearch.test.OpenSearchTestCase; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; + +public class CorrelationAlertServiceTests extends OpenSearchTestCase { + + public void testGetActiveAlerts() { + // Mock setup + Client client = mock(Client.class); + NamedXContentRegistry xContentRegistry = mock(NamedXContentRegistry.class); + CorrelationAlertService alertsService = spy(new CorrelationAlertService(client, xContentRegistry)); + + + // Fake data + String ruleId = "correlation_rule_id_123"; + long currentTime = System.currentTimeMillis(); + + // Define a fake correlation alert + CorrelationAlert correlationAlert = new CorrelationAlert( + Collections.emptyList(), + ruleId, + "mock-rule", + UUID.randomUUID().toString(), + 1L, + 1, + null, + "mock-trigger", + Alert.State.ACTIVE, + Instant.ofEpochMilli(currentTime).minusMillis(1000L), + Instant.ofEpochMilli(currentTime).plusMillis(1000L), + null, + null, + "high", + new ArrayList<>() + ); + + List correlationAlerts = Collections.singletonList(correlationAlert); + + // Call getActiveAlerts + alertsService.getActiveAlerts(ruleId, currentTime, new ActionListener() { + @Override + public void onResponse(CorrelationAlertsList correlationAlertsList) { + // Assertion + assertEquals(correlationAlerts.size(), correlationAlertsList.getCorrelationAlertList().size()); + + // Additional assertions can be added here to verify specific fields or states + CorrelationAlert returnedAlert = correlationAlertsList.getCorrelationAlertList().get(0); + assertEquals(correlationAlert.getId(), returnedAlert.getId()); + assertEquals(correlationAlert.getCorrelationRuleId(), returnedAlert.getCorrelationRuleId()); + assertEquals(correlationAlert.getStartTime(), returnedAlert.getStartTime()); + assertEquals(correlationAlert.getEndTime(), returnedAlert.getEndTime()); + } + + @Override + public void onFailure(Exception e) { + + } + }); + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/correlation/alerts/CorrelationAlertsRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/correlation/alerts/CorrelationAlertsRestApiIT.java new file mode 100644 index 000000000..f66f7cfe7 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/correlation/alerts/CorrelationAlertsRestApiIT.java @@ -0,0 +1,284 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.correlation.alerts; + +import org.apache.http.entity.StringEntity; +import org.apache.http.message.BasicHeader; +import org.junit.Assert; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.commons.alerting.model.CorrelationAlert; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.search.SearchHit; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.securityanalytics.model.DetectorInput; +import org.opensearch.securityanalytics.model.DetectorRule; +import org.opensearch.securityanalytics.model.DetectorTrigger; +import static org.opensearch.securityanalytics.TestHelpers.cloudtrailMappings; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailDoc; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailRuleForCorrelations; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndTriggersAndType; +import static org.opensearch.securityanalytics.TestHelpers.randomDoc; +import static org.opensearch.securityanalytics.TestHelpers.randomVpcFlowDoc; +import org.opensearch.test.rest.OpenSearchRestTestCase; +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; + + +public class CorrelationAlertsRestApiIT extends SecurityAnalyticsRestTestCase { + + public void testGetCorrelationAlertsAPI() throws IOException, InterruptedException { + LogIndices indices = createIndices(); + + String vpcFlowMonitorId = createVpcFlowDetector(indices.vpcFlowsIndex); + String testWindowsMonitorId = createTestWindowsDetector(indices.windowsIndex); + + createNetworkToAdLdapToWindowsRuleWithTrigger(indices); + Thread.sleep(5000); + + indexDoc(indices.windowsIndex, "2", randomDoc()); + Response executeResponse = executeAlertingMonitor(testWindowsMonitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + Thread.sleep(5000); + indexDoc(indices.vpcFlowsIndex, "1", randomVpcFlowDoc()); + executeResponse = executeAlertingMonitor(vpcFlowMonitorId, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(1, noOfSigmaRuleMatches); + + Thread.sleep(5000); + + OpenSearchRestTestCase.waitUntil( + () -> { + try { + Long endTime = System.currentTimeMillis(); + Request request = new Request("GET", "/_plugins/_security_analytics/correlationAlerts"); + Response response = client().performRequest(request); + + Map responseMap = entityAsMap(response); + List correlationAlerts = (List) responseMap.get("correlationAlerts"); + if (correlationAlerts.size() == 1) { + Assert.assertEquals(correlationAlerts.get(0).getTriggerName(), "Trigger 1"); + Assert.assertTrue(true); + return true; + } + return false; + } catch (Exception ex) { + return false; + } + }, + 2, TimeUnit.MINUTES + ); + } + + public void testGetCorrelationAlertsByRuleIdAPI() throws IOException, InterruptedException { + String index = createTestIndex("cloudtrail", cloudtrailMappings()); + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{\n" + + " \"index_name\": \"" + index + "\",\n" + + " \"rule_topic\": \"cloudtrail\",\n" + + " \"partial\": true,\n" + + " \"alias_mappings\": {\n" + + " \"properties\": {\n" + + " \"aws.cloudtrail.event_name\": {\n" + + " \"path\": \"Records.eventName\",\n" + + " \"type\": \"alias\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(RestStatus.OK.getStatus(), response.getStatusLine().getStatusCode()); + + String rule1 = randomCloudtrailRuleForCorrelations("CreateUser"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"), + new StringEntity(rule1), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + String createdId1 = responseBody.get("_id").toString(); + + String rule2 = randomCloudtrailRuleForCorrelations("DeleteUser"); + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"), + new StringEntity(rule2), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + responseBody = asMap(createResponse); + String createdId2 = responseBody.get("_id").toString(); + + createCloudtrailFieldBasedRuleWithTrigger(index, "requestParameters.userName", null); + + Detector cloudtrailDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("cloudtrail detector for security analytics", List.of(index), + List.of(new DetectorRule(createdId1), new DetectorRule(createdId2)), + List.of())), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("cloudtrail"), List.of(), List.of(), List.of(), List.of(), List.of())), "cloudtrail"); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(cloudtrailDetector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + String createdId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", randomCloudtrailDoc("Richard", "CreateUser")); + executeAlertingMonitor(monitorId, Collections.emptyMap()); + Thread.sleep(1000); + indexDoc(index, "4", randomCloudtrailDoc("deysubho", "CreateUser")); + executeAlertingMonitor(monitorId, Collections.emptyMap()); + Thread.sleep(1000); + + indexDoc(index, "2", randomCloudtrailDoc("Richard", "DeleteUser")); + executeAlertingMonitor(monitorId, Collections.emptyMap()); + + Thread.sleep(5000); + + OpenSearchRestTestCase.waitUntil( + () -> { + try { + Request restRequest = new Request("GET", "/_plugins/_security_analytics/correlationAlerts?correlation_rule_id=correlation-rule-1"); + Response restResponse = client().performRequest(restRequest); + + Map responseMap = entityAsMap(restResponse); + int totalAlerts = (int) responseMap.get("total_alerts"); + if (totalAlerts == 1) { + Assert.assertTrue(true); + return true; + } + return false; + } catch (Exception ex) { + return false; + } + }, + 2, TimeUnit.MINUTES + ); + } + + public void testGetCorrelationAlertsAcknowledgeAPI() throws IOException, InterruptedException { + String index = createTestIndex("cloudtrail", cloudtrailMappings()); + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{\n" + + " \"index_name\": \"" + index + "\",\n" + + " \"rule_topic\": \"cloudtrail\",\n" + + " \"partial\": true,\n" + + " \"alias_mappings\": {\n" + + " \"properties\": {\n" + + " \"aws.cloudtrail.event_name\": {\n" + + " \"path\": \"Records.eventName\",\n" + + " \"type\": \"alias\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(RestStatus.OK.getStatus(), response.getStatusLine().getStatusCode()); + + String rule1 = randomCloudtrailRuleForCorrelations("CreateUser"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"), + new StringEntity(rule1), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + String createdId1 = responseBody.get("_id").toString(); + + String rule2 = randomCloudtrailRuleForCorrelations("DeleteUser"); + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"), + new StringEntity(rule2), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + responseBody = asMap(createResponse); + String createdId2 = responseBody.get("_id").toString(); + + createCloudtrailFieldBasedRuleWithTrigger(index, "requestParameters.userName", null); + + Detector cloudtrailDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("cloudtrail detector for security analytics", List.of(index), + List.of(new DetectorRule(createdId1), new DetectorRule(createdId2)), + List.of())), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("cloudtrail"), List.of(), List.of(), List.of(), List.of(), List.of())), "cloudtrail"); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(cloudtrailDetector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + String createdId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", randomCloudtrailDoc("Richard", "CreateUser")); + executeAlertingMonitor(monitorId, Collections.emptyMap()); + Thread.sleep(1000); + indexDoc(index, "4", randomCloudtrailDoc("John", "CreateUser")); + executeAlertingMonitor(monitorId, Collections.emptyMap()); + Thread.sleep(1000); + + indexDoc(index, "2", randomCloudtrailDoc("Richard", "DeleteUser")); + executeAlertingMonitor(monitorId, Collections.emptyMap()); + + Thread.sleep(5000); + OpenSearchRestTestCase.waitUntil( + () -> { + try { + Request request1 = new Request("GET", "/_plugins/_security_analytics/correlationAlerts"); + Response getCorrelationAlertResp = client().performRequest(request1); + Map responseGetCorrelationAlertMap = entityAsMap(getCorrelationAlertResp); + List correlationAlerts = (List) responseGetCorrelationAlertMap.get("correlationAlerts"); + // Execute CreateMappingsAction to add alias mapping for index + Thread.sleep(2000); + Request restRequest = new Request("POST", "/_plugins/_security_analytics/_acknowledge/correlationAlerts"); + restRequest.setJsonEntity( + "{\"alertIds\": [\"" + correlationAlerts.get(0).getId() + "\"]}" + ); + Response restResponse = client().performRequest(restRequest); + Map responseMap = entityAsMap(restResponse); + List results = (List) responseMap.get("acknowledged"); + if (results.size() == 1) { + Assert.assertTrue(true); + return true; + } + return false; + } catch (Exception ex) { + return false; + } + }, + 2, TimeUnit.MINUTES + ); + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java index db282ec2c..ab06b8d30 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java @@ -265,7 +265,7 @@ public void testGetFindings_byDetectorType_success() throws IOException { noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); Assert.assertEquals(1, noOfSigmaRuleMatches); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Call GetFindings API for first detector Map params = new HashMap<>(); @@ -375,7 +375,7 @@ public void testGetAllFindings_success() throws IOException { noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); // Assert.assertEquals(1, noOfSigmaRuleMatches); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Call GetFindings API for all the detectors Map params = new HashMap<>(); @@ -594,7 +594,7 @@ public void testGetFindings_bySeverity_success() throws IOException { noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); Assert.assertEquals(1, noOfSigmaRuleMatches); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Call GetFindings API for first detector by severity Map params = new HashMap<>(); @@ -715,7 +715,7 @@ public void testGetFindings_bySearchString_success() throws IOException { noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); Assert.assertEquals(1, noOfSigmaRuleMatches); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Call GetFindings API for first detector by searchString 'high' Map params = new HashMap<>(); @@ -831,7 +831,7 @@ public void testGetFindings_byStartTimeAndEndTime_success() throws IOException { int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); Assert.assertEquals(1, noOfSigmaRuleMatches); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Call GetFindings API for first detector by startTime and endTime Map params = new HashMap<>(); params.put("startTime", String.valueOf(startTime1.toEpochMilli())); @@ -842,7 +842,7 @@ public void testGetFindings_byStartTimeAndEndTime_success() throws IOException { Map getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); Instant startTime2 = Instant.now(); // execute monitor 2 executeResponse = executeAlertingMonitor(monitorId2, Collections.emptyMap()); @@ -1223,7 +1223,7 @@ public void testCreateDetectorWithNotCondition_verifyFindingsAndNoFindings_succe " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(1, response.getHits().getTotalHits().value); @@ -1343,7 +1343,7 @@ public void testGetFindings_rolloverByMaxDoc_short_retention_success() throws IO // Call GetFindings API Map params = new HashMap<>(); params.put("detector_id", detectorId); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); Map getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); @@ -1372,7 +1372,7 @@ public void testGetFindings_rolloverByMaxDoc_short_retention_success() throws IO noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); Assert.assertEquals(5, noOfSigmaRuleMatches); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); diff --git a/src/test/java/org/opensearch/securityanalytics/findings/SecureFindingRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/findings/SecureFindingRestApiIT.java index c24b344bd..b5eb8ede2 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/SecureFindingRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/SecureFindingRestApiIT.java @@ -263,7 +263,7 @@ public void testGetFindings_byDetectorType_success() throws IOException { noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); Assert.assertEquals(5, noOfSigmaRuleMatches); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // try to do get finding as a user with read access diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java index 7389550c9..d389797c5 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java @@ -722,7 +722,7 @@ public void testCreateMappings_withIndexPattern_differentMappings_indexTemplateC createIndex(indexName1, Settings.EMPTY, null); createIndex(indexName2, Settings.EMPTY, null); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Insert sample docs String sampleDoc1 = "{" + @@ -737,7 +737,7 @@ public void testCreateMappings_withIndexPattern_differentMappings_indexTemplateC indexDoc(indexName1, "1", sampleDoc1); indexDoc(indexName2, "1", sampleDoc2); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Execute CreateMappingsAction to add alias mapping for index createMappingsAPI(indexPattern, "netflow"); @@ -800,7 +800,7 @@ public void testCreateMappings_withIndexPattern_indexTemplate_createAndUpdate_su createIndex(indexName1, Settings.EMPTY, null); createIndex(indexName2, Settings.EMPTY, null); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Insert sample doc String sampleDoc1 = "{" + @@ -811,7 +811,7 @@ public void testCreateMappings_withIndexPattern_indexTemplate_createAndUpdate_su indexDoc(indexName1, "1", sampleDoc1); indexDoc(indexName2, "1", sampleDoc1); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Execute CreateMappingsAction to add alias mapping for index createMappingsAPI(indexPattern, "netflow"); @@ -887,7 +887,7 @@ public void testCreateMappings_withIndexPattern_oneNoMappings_failure() throws I createIndex(indexName1, Settings.EMPTY, null); createIndex(indexName2, Settings.EMPTY, null); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Insert sample docs String sampleDoc1 = "{" + @@ -897,7 +897,7 @@ public void testCreateMappings_withIndexPattern_oneNoMappings_failure() throws I "}"; indexDoc(indexName1, "1", sampleDoc1); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Execute CreateMappingsAction to add alias mapping for index try { @@ -1111,7 +1111,7 @@ public void testCreateMappings_withIndexPattern_success() throws IOException { createIndex(indexName1, Settings.EMPTY, null); createIndex(indexName2, Settings.EMPTY, null); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Insert sample doc String sampleDoc = "{" + @@ -1124,7 +1124,7 @@ public void testCreateMappings_withIndexPattern_success() throws IOException { indexDoc(indexName1, "1", sampleDoc); indexDoc(indexName2, "1", sampleDoc); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Execute CreateMappingsAction to add alias mapping for index Request request = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); @@ -1150,7 +1150,7 @@ public void testCreateMappings_withIndexPattern_conflictingTemplates_success() t createIndex(indexName1, Settings.EMPTY, null); createIndex(indexName2, Settings.EMPTY, null); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Insert sample doc String sampleDoc = "{" + @@ -1161,7 +1161,7 @@ public void testCreateMappings_withIndexPattern_conflictingTemplates_success() t indexDoc(indexName1, "1", sampleDoc); indexDoc(indexName2, "1", sampleDoc); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Execute CreateMappingsAction with first index pattern createMappingsAPI(indexPattern1, "netflow"); @@ -1205,7 +1205,7 @@ public void testCreateMappings_withIndexPattern_conflictingTemplates_failure_1() createIndex(indexName1, Settings.EMPTY, null); createIndex(indexName2, Settings.EMPTY, null); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Insert sample doc String sampleDoc = "{" + @@ -1216,7 +1216,7 @@ public void testCreateMappings_withIndexPattern_conflictingTemplates_failure_1() indexDoc(indexName1, "1", sampleDoc); indexDoc(indexName2, "1", sampleDoc); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Execute CreateMappingsAction with first index pattern createMappingsAPI(indexPattern1, "netflow"); @@ -1244,7 +1244,7 @@ public void testCreateMappings_withIndexPattern_conflictingTemplates_failure_2() createIndex(indexName1, Settings.EMPTY, null); createIndex(indexName2, Settings.EMPTY, null); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Insert sample doc String sampleDoc = "{" + @@ -1255,7 +1255,7 @@ public void testCreateMappings_withIndexPattern_conflictingTemplates_failure_2() indexDoc(indexName1, "1", sampleDoc); indexDoc(indexName2, "1", sampleDoc); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // User-create template with conflicting pattern but higher priority @@ -1279,7 +1279,7 @@ public void testCreateMappings_withIndexPattern_oneNoMatches_success() throws IO createIndex(indexName1, Settings.EMPTY, null); createIndex(indexName2, Settings.EMPTY, null); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Insert sample docs String sampleDoc1 = "{" + @@ -1294,7 +1294,7 @@ public void testCreateMappings_withIndexPattern_oneNoMatches_success() throws IO indexDoc(indexName1, "1", sampleDoc1); indexDoc(indexName2, "1", sampleDoc2); - client().performRequest(new Request("POST", "_refresh")); + // client().performRequest(new Request("POST", "_refresh")); // Execute CreateMappingsAction to add alias mapping for index Request request = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); @@ -1381,8 +1381,8 @@ private void createSampleIndex(String indexName, Settings settings, String alias Response response = client().performRequest(indexRequest); assertEquals(HttpStatus.SC_CREATED, response.getStatusLine().getStatusCode()); // Refresh everything - response = client().performRequest(new Request("POST", "_refresh")); - assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + //response = client().performRequest(new Request("POST", "_refresh")); + //assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } private void createSampleWindex(String indexName) throws IOException { @@ -1444,8 +1444,8 @@ private void createSampleWindex(String indexName, Settings settings, String alia Response response = client().performRequest(indexRequest); assertEquals(HttpStatus.SC_CREATED, response.getStatusLine().getStatusCode()); // Refresh everything - response = client().performRequest(new Request("POST", "_refresh")); - assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + //response = client().performRequest(new Request("POST", "_refresh")); + //assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } private void createSampleDatastream(String datastreamName) throws IOException { @@ -1533,8 +1533,8 @@ private void createSampleDatastream(String datastreamName) throws IOException { response = client().performRequest(indexRequest); assertEquals(HttpStatus.SC_CREATED, response.getStatusLine().getStatusCode()); // Refresh everything - response = client().performRequest(new Request("POST", "_refresh")); - assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + //response = client().performRequest(new Request("POST", "_refresh")); + //assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } private void deleteDatastream(String datastreamName) throws IOException { @@ -1613,8 +1613,8 @@ public void testCreateDNSMapping() throws IOException{ }); // Refresh everything - response = client().performRequest(new Request("POST", "_refresh")); - assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + //response = client().performRequest(new Request("POST", "_refresh")); + //assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } @@ -1656,8 +1656,8 @@ public void testTraverseAndCopy() { " \"type\":\"keyword\"," + " \"ignore_above\":256" + " }" + - " }" + - " }" + + " }" + + " }" + " }" + " }" + "}"; @@ -1723,7 +1723,7 @@ public void testAzureMappings() throws IOException { Detector detector = randomDetectorWithInputs(List.of(input), "azure"); createDetector(detector); - List hits = executeSearch(".opensearch-sap-azure-detectors-queries-000001", matchAllSearchBody); + List hits = executeSearch(".opensearch-sap-azure-detectors-queries-*", matchAllSearchBody); Assert.assertEquals(127, hits.size()); } @@ -1748,7 +1748,7 @@ public void testADLDAPMappings() throws IOException { Detector detector = randomDetectorWithInputs(List.of(input), "ad_ldap"); createDetector(detector); - List hits = executeSearch(".opensearch-sap-ad_ldap-detectors-queries-000001", matchAllSearchBody); + List hits = executeSearch(".opensearch-sap-ad_ldap-detectors-queries-*", matchAllSearchBody); Assert.assertEquals(11, hits.size()); } @@ -1773,7 +1773,7 @@ public void testCloudtrailMappings() throws IOException { Detector detector = randomDetectorWithInputs(List.of(input), "cloudtrail"); createDetector(detector); - List hits = executeSearch(".opensearch-sap-cloudtrail-detectors-queries-000001", matchAllSearchBody); + List hits = executeSearch(".opensearch-sap-cloudtrail-detectors-queries-*", matchAllSearchBody); Assert.assertEquals(39, hits.size()); } @@ -1798,7 +1798,7 @@ public void testS3Mappings() throws IOException { Detector detector = randomDetectorWithInputs(List.of(input), "s3"); createDetector(detector); - List hits = executeSearch(".opensearch-sap-s3-detectors-queries-000001", matchAllSearchBody); + List hits = executeSearch(".opensearch-sap-s3-detectors-queries-*", matchAllSearchBody); Assert.assertEquals(1, hits.size()); } @@ -1825,7 +1825,7 @@ public void testWAFMappings() throws IOException { Detector detector = randomDetectorWithInputs(List.of(input), "waf"); createDetector(detector); - List hits = executeSearch(".opensearch-sap-waf-detectors-queries-000001", matchAllSearchBody); + List hits = executeSearch(".opensearch-sap-waf-detectors-queries-*", matchAllSearchBody); Assert.assertEquals(5, hits.size()); } diff --git a/src/test/java/org/opensearch/securityanalytics/model/DetailedSTIX2IOCDtoTests.java b/src/test/java/org/opensearch/securityanalytics/model/DetailedSTIX2IOCDtoTests.java new file mode 100644 index 000000000..6fd26291d --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/model/DetailedSTIX2IOCDtoTests.java @@ -0,0 +1,37 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.model; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +import static org.opensearch.securityanalytics.TestHelpers.parser; +import static org.opensearch.securityanalytics.util.STIX2IOCGenerator.assertEqualIocDtos; +import static org.opensearch.securityanalytics.util.STIX2IOCGenerator.randomIocDto; +import static org.opensearch.securityanalytics.util.STIX2IOCGenerator.toJsonString; + +public class DetailedSTIX2IOCDtoTests extends OpenSearchTestCase { + public void testAsStream() throws IOException { + long numFindings = randomLongBetween(0, 100); + DetailedSTIX2IOCDto ioc = new DetailedSTIX2IOCDto(randomIocDto(), numFindings); + BytesStreamOutput out = new BytesStreamOutput(); + ioc.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + DetailedSTIX2IOCDto newIoc = new DetailedSTIX2IOCDto(sin); + assertEqualIocDtos(ioc, newIoc); + } + + public void testParseFunction() throws IOException { + long numFindings = randomLongBetween(0, 100); + DetailedSTIX2IOCDto ioc = new DetailedSTIX2IOCDto(randomIocDto(), numFindings); + String json = toJsonString(ioc); + DetailedSTIX2IOCDto newIoc = DetailedSTIX2IOCDto.parse(parser(json), ioc.getIoc().getId(), ioc.getIoc().getVersion()); + assertEqualIocDtos(ioc, newIoc); + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/model/IocFindingTests.java b/src/test/java/org/opensearch/securityanalytics/model/IocFindingTests.java new file mode 100644 index 000000000..444fc64f9 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/model/IocFindingTests.java @@ -0,0 +1,80 @@ +package org.opensearch.securityanalytics.model; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; +import org.opensearch.securityanalytics.model.threatintel.IocWithFeeds; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.util.List; + +import static org.opensearch.securityanalytics.TestHelpers.toJsonString; + +public class IocFindingTests extends OpenSearchTestCase { + + public void testIoCMatchAsAStream() throws IOException { + IocFinding iocFinding = getRandomIoCMatch(); + String jsonString = toJsonString(iocFinding); + BytesStreamOutput out = new BytesStreamOutput(); + iocFinding.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + IocFinding newIocFinding = new IocFinding(sin); + assertEquals(iocFinding.getId(), newIocFinding.getId()); + assertEquals(iocFinding.getMonitorId(), newIocFinding.getMonitorId()); + assertEquals(iocFinding.getMonitorName(), newIocFinding.getMonitorName()); + assertEquals(iocFinding.getIocValue(), newIocFinding.getIocValue()); + assertEquals(iocFinding.getIocType(), newIocFinding.getIocType()); + assertEquals(iocFinding.getTimestamp(), newIocFinding.getTimestamp()); + assertEquals(iocFinding.getExecutionId(), newIocFinding.getExecutionId()); + assertTrue(iocFinding.getFeedIds().containsAll(newIocFinding.getFeedIds())); + assertTrue(iocFinding.getRelatedDocIds().containsAll(newIocFinding.getRelatedDocIds())); + } + + public void testIoCMatchParse() throws IOException { + String iocMatchString = "{ \"id\": \"exampleId123\", \"related_doc_ids\": [\"relatedDocId1\", " + + "\"relatedDocId2\"], \"feed_ids\": [\"feedId1\", \"feedId2\"], \"monitor_id\":" + + " \"scanJob123\", \"monitor_name\": \"Example Scan Job\", \"ioc_value\": \"exampleIocValue\", " + + "\"ioc_type\": \"exampleIocType\", \"timestamp\": 1620912896000, \"execution_id\": \"execution123\" }"; + IocFinding iocFinding = IocFinding.parse((getParser(iocMatchString))); + BytesStreamOutput out = new BytesStreamOutput(); + iocFinding.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + IocFinding newIocFinding = new IocFinding(sin); + assertEquals(iocFinding.getId(), newIocFinding.getId()); + assertEquals(iocFinding.getMonitorId(), newIocFinding.getMonitorId()); + assertEquals(iocFinding.getMonitorName(), newIocFinding.getMonitorName()); + assertEquals(iocFinding.getIocValue(), newIocFinding.getIocValue()); + assertEquals(iocFinding.getIocType(), newIocFinding.getIocType()); + assertEquals(iocFinding.getTimestamp(), newIocFinding.getTimestamp()); + assertEquals(iocFinding.getExecutionId(), newIocFinding.getExecutionId()); + assertTrue(iocFinding.getFeedIds().containsAll(newIocFinding.getFeedIds())); + assertTrue(iocFinding.getRelatedDocIds().containsAll(newIocFinding.getRelatedDocIds())); + } + + public XContentParser getParser(String xc) throws IOException { + XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); + parser.nextToken(); + return parser; + + } + + private static IocFinding getRandomIoCMatch() { + return new IocFinding( + randomAlphaOfLength(10), + List.of(randomAlphaOfLength(10), randomAlphaOfLength(10)), + List.of(new IocWithFeeds(randomAlphaOfLength(10),randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10))), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + Instant.now(), + randomAlphaOfLength(10)); + } + + +} diff --git a/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigDtoTests.java b/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigDtoTests.java new file mode 100644 index 000000000..e5b2ed7c5 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigDtoTests.java @@ -0,0 +1,109 @@ +package org.opensearch.securityanalytics.model; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; +import org.opensearch.securityanalytics.threatIntel.model.S3Source; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; + +import static org.opensearch.securityanalytics.TestHelpers.randomSATIFSourceConfigDto; + +public class SATIFSourceConfigDtoTests extends OpenSearchTestCase { + + public void testAsStream() throws IOException { + SATIFSourceConfigDto saTifSourceConfigDto = randomSATIFSourceConfigDto(); + BytesStreamOutput out = new BytesStreamOutput(); + saTifSourceConfigDto.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + SATIFSourceConfigDto newSaTifSourceConfigDto = new SATIFSourceConfigDto(sin); + assertEqualsSaTifSourceConfigDtos(saTifSourceConfigDto, newSaTifSourceConfigDto); + } + + public void testParseFunction() throws IOException { + SATIFSourceConfigDto saTifSourceConfigDto = randomSATIFSourceConfigDto(); + String json = toJsonString(saTifSourceConfigDto); + SATIFSourceConfigDto newSaTifSourceConfigDto = SATIFSourceConfigDto.parse(getParser(json), saTifSourceConfigDto.getId(), null); + assertEqualsSaTifSourceConfigDtos(saTifSourceConfigDto, newSaTifSourceConfigDto); + } + + public void testParseFunctionWithNullValues() throws IOException { + // Source config with invalid name and format + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + "randomId", + null, + null, + null, + SourceConfigType.S3_CUSTOM, + null, + null, + null, + new S3Source("bucket", "objectkey", "region", "rolearn"), + null, + null, + new org.opensearch.jobscheduler.spi.schedule.IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS), + null, + null, + null, + null, + true, + List.of("ip"), + true + ); + String json = toJsonString(saTifSourceConfigDto); + SATIFSourceConfigDto newSaTifSourceConfigDto = SATIFSourceConfigDto.parse(getParser(json), saTifSourceConfigDto.getId(), null); + assertEqualsSaTifSourceConfigDtos(saTifSourceConfigDto, newSaTifSourceConfigDto); + } + + public XContentParser getParser(String xc) throws IOException { + XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); + parser.nextToken(); + return parser; + + } + private String toJsonString(SATIFSourceConfigDto saTifSourceConfigDto) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder = saTifSourceConfigDto.toXContent(builder, ToXContent.EMPTY_PARAMS); + return BytesReference.bytes(builder).utf8ToString(); + } + + private void assertEqualsSaTifSourceConfigDtos(SATIFSourceConfigDto saTifSourceConfigDto, SATIFSourceConfigDto newSaTifSourceConfigDto) { + assertEquals(saTifSourceConfigDto.getId(), newSaTifSourceConfigDto.getId()); + assertEquals(saTifSourceConfigDto.getVersion(), newSaTifSourceConfigDto.getVersion()); + assertEquals(saTifSourceConfigDto.getName(), newSaTifSourceConfigDto.getName()); + assertEquals(saTifSourceConfigDto.getFormat(), newSaTifSourceConfigDto.getFormat()); + assertEquals(saTifSourceConfigDto.getType(), newSaTifSourceConfigDto.getType()); + assertEquals(saTifSourceConfigDto.getDescription(), newSaTifSourceConfigDto.getDescription()); + assertEquals(saTifSourceConfigDto.getCreatedByUser(), newSaTifSourceConfigDto.getCreatedByUser()); + assertEquals(saTifSourceConfigDto.getCreatedAt().toEpochMilli(), newSaTifSourceConfigDto.getCreatedAt().toEpochMilli()); + S3Source source = (S3Source)saTifSourceConfigDto.getSource(); + S3Source newSource = (S3Source)newSaTifSourceConfigDto.getSource(); + assertEquals(source.getBucketName(), newSource.getBucketName()); + assertEquals(source.getRegion(), newSource.getRegion()); + assertEquals(source.getObjectKey(), newSource.getObjectKey()); + assertEquals(source.getRoleArn(), newSource.getRoleArn()); + assertEquals(saTifSourceConfigDto.getEnabledTime().toEpochMilli(), newSaTifSourceConfigDto.getEnabledTime().toEpochMilli()); + assertEquals(saTifSourceConfigDto.getLastUpdateTime().toEpochMilli(), newSaTifSourceConfigDto.getLastUpdateTime().toEpochMilli()); + assertEquals(((IntervalSchedule)saTifSourceConfigDto.getSchedule()).getStartTime().toEpochMilli(), ((IntervalSchedule)newSaTifSourceConfigDto.getSchedule()).getStartTime().toEpochMilli()); + assertEquals(((IntervalSchedule)saTifSourceConfigDto.getSchedule()).getInterval(), ((IntervalSchedule)newSaTifSourceConfigDto.getSchedule()).getInterval()); + assertEquals(((IntervalSchedule)saTifSourceConfigDto.getSchedule()).getUnit(), ((IntervalSchedule)newSaTifSourceConfigDto.getSchedule()).getUnit()); + assertEquals(saTifSourceConfigDto.getState(), newSaTifSourceConfigDto.getState()); + assertEquals(saTifSourceConfigDto.getRefreshType(), newSaTifSourceConfigDto.getRefreshType()); + assertEquals(saTifSourceConfigDto.getLastRefreshedTime(), newSaTifSourceConfigDto.getLastRefreshedTime()); + assertEquals(saTifSourceConfigDto.isEnabled(), newSaTifSourceConfigDto.isEnabled()); + assertEquals(saTifSourceConfigDto.getIocTypes(), newSaTifSourceConfigDto.getIocTypes()); + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigTests.java b/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigTests.java new file mode 100644 index 000000000..8fa8ec395 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigTests.java @@ -0,0 +1,117 @@ +package org.opensearch.securityanalytics.model; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; +import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; +import org.opensearch.securityanalytics.threatIntel.model.S3Source; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; + +import static org.opensearch.securityanalytics.TestHelpers.randomSATIFSourceConfig; + +public class SATIFSourceConfigTests extends OpenSearchTestCase { + + public void testAsStream() throws IOException { + SATIFSourceConfig saTifSourceConfig = randomSATIFSourceConfig(); + BytesStreamOutput out = new BytesStreamOutput(); + saTifSourceConfig.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + SATIFSourceConfig newSaTifSourceConfig = new SATIFSourceConfig(sin); + assertEqualsSaTifSourceConfigs(saTifSourceConfig, newSaTifSourceConfig); + } + + public void testParseFunction() throws IOException { + SATIFSourceConfig saTifSourceConfig = randomSATIFSourceConfig(); + String json = toJsonString(saTifSourceConfig); + SATIFSourceConfig newSaTifSourceConfig = SATIFSourceConfig.parse(getParser(json), saTifSourceConfig.getId(), null); + assertEqualsSaTifSourceConfigs(saTifSourceConfig, newSaTifSourceConfig); + } + + public void testParseFunctionWithNullValues() throws IOException { + // Source config with invalid name and format + SATIFSourceConfig saTifSourceConfig = new SATIFSourceConfig( + null, + null, + null, + null, + SourceConfigType.S3_CUSTOM, + null, + null, + null, + new S3Source("bucket", "objectkey", "region", "rolearn"), + null, + null, + new org.opensearch.jobscheduler.spi.schedule.IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS), + null, + null, + null, + null, + true, + new DefaultIocStoreConfig(List.of(new DefaultIocStoreConfig.IocToIndexDetails(new IOCType(IOCType.DOMAIN_NAME_TYPE), "indexPattern", "writeIndex"))), + List.of("ip"), + true + ); + String json = toJsonString(saTifSourceConfig); + SATIFSourceConfig newSaTifSourceConfig = SATIFSourceConfig.parse(getParser(json), saTifSourceConfig.getId(), null); + assertEqualsSaTifSourceConfigs(saTifSourceConfig, newSaTifSourceConfig); + } + + public XContentParser getParser(String xc) throws IOException { + XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); + parser.nextToken(); + return parser; + + } + private String toJsonString(SATIFSourceConfig saTifSourceConfig) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder = saTifSourceConfig.toXContent(builder, ToXContent.EMPTY_PARAMS); + return BytesReference.bytes(builder).utf8ToString(); + } + + private void assertEqualsSaTifSourceConfigs(SATIFSourceConfig saTifSourceConfig, SATIFSourceConfig newSaTifSourceConfig) { + assertEquals(saTifSourceConfig.getId(), newSaTifSourceConfig.getId()); + assertEquals(saTifSourceConfig.getVersion(), newSaTifSourceConfig.getVersion()); + assertEquals(saTifSourceConfig.getName(), newSaTifSourceConfig.getName()); + assertEquals(saTifSourceConfig.getFormat(), newSaTifSourceConfig.getFormat()); + assertEquals(saTifSourceConfig.getType(), newSaTifSourceConfig.getType()); + assertEquals(saTifSourceConfig.getDescription(), newSaTifSourceConfig.getDescription()); + assertEquals(saTifSourceConfig.getCreatedByUser(), newSaTifSourceConfig.getCreatedByUser()); + assertEquals(saTifSourceConfig.getCreatedAt().toEpochMilli(), newSaTifSourceConfig.getCreatedAt().toEpochMilli()); + S3Source source = (S3Source)saTifSourceConfig.getSource(); + S3Source newSource = (S3Source)newSaTifSourceConfig.getSource(); + assertEquals(source.getBucketName(), newSource.getBucketName()); + assertEquals(source.getRegion(), newSource.getRegion()); + assertEquals(source.getObjectKey(), newSource.getObjectKey()); + assertEquals(source.getRoleArn(), newSource.getRoleArn()); + assertEquals(saTifSourceConfig.getEnabledTime().toEpochMilli(), newSaTifSourceConfig.getEnabledTime().toEpochMilli()); + assertEquals(saTifSourceConfig.getLastUpdateTime().toEpochMilli(), newSaTifSourceConfig.getLastUpdateTime().toEpochMilli()); + assertEquals(((IntervalSchedule)saTifSourceConfig.getSchedule()).getStartTime().toEpochMilli(), ((IntervalSchedule) newSaTifSourceConfig.getSchedule()).getStartTime().toEpochMilli()); + assertEquals(((IntervalSchedule)saTifSourceConfig.getSchedule()).getInterval(), ((IntervalSchedule)newSaTifSourceConfig.getSchedule()).getInterval()); + assertEquals(((IntervalSchedule)saTifSourceConfig.getSchedule()).getUnit(), ((IntervalSchedule) newSaTifSourceConfig.getSchedule()).getUnit()); + assertEquals(saTifSourceConfig.getState(), newSaTifSourceConfig.getState()); + assertEquals(saTifSourceConfig.getRefreshType(), newSaTifSourceConfig.getRefreshType()); + assertEquals(saTifSourceConfig.getLastRefreshedTime(), newSaTifSourceConfig.getLastRefreshedTime()); + assertEquals(saTifSourceConfig.isEnabled(), newSaTifSourceConfig.isEnabled()); + DefaultIocStoreConfig iocStoreConfig = (DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig(); + DefaultIocStoreConfig newIocStoreConfig = (DefaultIocStoreConfig) newSaTifSourceConfig.getIocStoreConfig(); + assertEquals(iocStoreConfig.getIocToIndexDetails().get(0).getIocType().toString(), newIocStoreConfig.getIocToIndexDetails().get(0).getIocType().toString()); + assertEquals(iocStoreConfig.getIocToIndexDetails().get(0).getIndexPattern(), newIocStoreConfig.getIocToIndexDetails().get(0).getIndexPattern()); + assertEquals(iocStoreConfig.getIocToIndexDetails().get(0).getActiveIndex(), newIocStoreConfig.getIocToIndexDetails().get(0).getActiveIndex()); + assertEquals(saTifSourceConfig.getIocTypes(), newSaTifSourceConfig.getIocTypes()); + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/model/STIX2IOCDtoTests.java b/src/test/java/org/opensearch/securityanalytics/model/STIX2IOCDtoTests.java new file mode 100644 index 000000000..110d75d50 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/model/STIX2IOCDtoTests.java @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.model; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +import static org.opensearch.securityanalytics.TestHelpers.parser; +import static org.opensearch.securityanalytics.util.STIX2IOCGenerator.assertEqualIocDtos; +import static org.opensearch.securityanalytics.util.STIX2IOCGenerator.randomIocDto; +import static org.opensearch.securityanalytics.util.STIX2IOCGenerator.toJsonString; + +public class STIX2IOCDtoTests extends OpenSearchTestCase { + public void testAsStream() throws IOException { + STIX2IOCDto ioc = randomIocDto(); + BytesStreamOutput out = new BytesStreamOutput(); + ioc.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + STIX2IOCDto newIoc = new STIX2IOCDto(sin); + assertEqualIocDtos(ioc, newIoc); + } + + public void testParseFunction() throws IOException { + STIX2IOCDto ioc = randomIocDto(); + String json = toJsonString(ioc); + STIX2IOCDto newIoc = STIX2IOCDto.parse(parser(json), ioc.getId(), ioc.getVersion()); + assertEqualIocDtos(ioc, newIoc); + } + + public void testParseFunction_invalidType() throws IOException { + // Execute test case for each IOCType + for (String type : IOCType.types) { + STIX2IOCDto ioc = randomIocDto(new IOCType(type)); + String json = toJsonString(ioc); + + // Replace the IOCType with a fake type + String fakeType = "fake" + type; + final String invalidJson = json.replace(type, fakeType); + + SecurityAnalyticsException exception = assertThrows(SecurityAnalyticsException.class, () -> STIX2IOCDto.parse(parser(invalidJson), ioc.getId(), ioc.getVersion())); + assertEquals(RestStatus.BAD_REQUEST, exception.status()); + + String expectedError = String.format( + "Couldn't parse IOC type '%s' while deserializing STIX2IOCDto with ID '%s': ", + fakeType, + ioc.getId() + ); + assertTrue(exception.getMessage().contains(expectedError)); + } + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/model/STIX2IOCTests.java b/src/test/java/org/opensearch/securityanalytics/model/STIX2IOCTests.java new file mode 100644 index 000000000..4323a03fb --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/model/STIX2IOCTests.java @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.model; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +import static org.opensearch.securityanalytics.TestHelpers.parser; +import static org.opensearch.securityanalytics.util.STIX2IOCGenerator.assertEqualIOCs; +import static org.opensearch.securityanalytics.util.STIX2IOCGenerator.randomIOC; +import static org.opensearch.securityanalytics.util.STIX2IOCGenerator.toJsonString; + +public class STIX2IOCTests extends OpenSearchTestCase { + public void testAsStream() throws IOException { + STIX2IOC ioc = randomIOC(); + BytesStreamOutput out = new BytesStreamOutput(); + ioc.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + STIX2IOC newIoc = new STIX2IOC(sin); + assertEqualIOCs(ioc, newIoc); + } + + public void testParseFunction() throws IOException { + STIX2IOC ioc = randomIOC(); + String json = toJsonString(ioc); + STIX2IOC newIoc = STIX2IOC.parse(parser(json), ioc.getId(), ioc.getVersion()); + assertEqualIOCs(ioc, newIoc); + } + + public void testParseFunction_invalidType() throws IOException { + // Execute test case for each IOCType + for (String type : IOCType.types) { + STIX2IOC ioc = randomIOC(new IOCType(type)); + String json = toJsonString(ioc); + + // Replace the IOCType with a fake type + String fakeType = "fake" + type; + final String invalidJson = json.replace(type, fakeType); + + SecurityAnalyticsException exception = assertThrows(SecurityAnalyticsException.class, () -> STIX2IOC.parse(parser(invalidJson), ioc.getId(), ioc.getVersion())); + assertEquals(RestStatus.BAD_REQUEST, exception.status()); + + String expectedError = String.format( + "Couldn't parse IOC type '%s' while deserializing STIX2IOC with ID '%s': ", + fakeType, + ioc.getId() + ); + assertTrue(exception.getMessage().contains(expectedError)); + } + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/model/WriteableTests.java b/src/test/java/org/opensearch/securityanalytics/model/WriteableTests.java index 2c5639c95..d50317333 100644 --- a/src/test/java/org/opensearch/securityanalytics/model/WriteableTests.java +++ b/src/test/java/org/opensearch/securityanalytics/model/WriteableTests.java @@ -77,7 +77,7 @@ public void testEmptyUserAsStream() throws IOException { public void testLogTypeAsStreamRawFieldOnly() throws IOException { LogType logType = new LogType( "1", "my_log_type", "description", false, - List.of(new LogType.Mapping("rawField", null, null)), + List.of(new LogType.Mapping("rawField", null, null, null)), List.of(new LogType.IocFields("ip", List.of("dst.ip"))) ); BytesStreamOutput out = new BytesStreamOutput(); @@ -94,7 +94,7 @@ public void testLogTypeAsStreamRawFieldOnly() throws IOException { public void testLogTypeAsStreamFull() throws IOException { LogType logType = new LogType( "1", "my_log_type", "description", false, - List.of(new LogType.Mapping("rawField", "some_ecs_field", "some_ocsf_field")), + List.of(new LogType.Mapping("rawField", "some_ecs_field", "some_ocsf_field", "some_ocsf11_field")), List.of(new LogType.IocFields("ip", List.of("dst.ip"))) ); BytesStreamOutput out = new BytesStreamOutput(); diff --git a/src/test/java/org/opensearch/securityanalytics/model/threatintel/ThreatIntelAlertTests.java b/src/test/java/org/opensearch/securityanalytics/model/threatintel/ThreatIntelAlertTests.java new file mode 100644 index 000000000..0e945d217 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/model/threatintel/ThreatIntelAlertTests.java @@ -0,0 +1,110 @@ +package org.opensearch.securityanalytics.model.threatintel; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.commons.alerting.model.Alert; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.util.Collections; +import java.util.List; + +public class ThreatIntelAlertTests extends OpenSearchTestCase { + + public void testAlertAsStream() throws IOException { + ThreatIntelAlert alert = getRandomAlert(); + BytesStreamOutput out = new BytesStreamOutput(); + alert.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + ThreatIntelAlert newThreatIntelAlert = new ThreatIntelAlert(sin); + asserts(alert, newThreatIntelAlert); + } + + private static void asserts(ThreatIntelAlert alert, ThreatIntelAlert newThreatIntelAlert) { + assertEquals(alert.getId(), newThreatIntelAlert.getId()); + assertEquals(alert.getErrorMessage(), newThreatIntelAlert.getErrorMessage()); + assertEquals(alert.getSeverity(), newThreatIntelAlert.getSeverity()); + assertEquals(alert.getSchemaVersion(), newThreatIntelAlert.getSchemaVersion()); + assertEquals(alert.getTriggerName(), newThreatIntelAlert.getTriggerName()); + assertEquals(alert.getTriggerId(), newThreatIntelAlert.getTriggerId()); + assertEquals(alert.getMonitorId(), newThreatIntelAlert.getMonitorId()); + assertEquals(alert.getMonitorName(), newThreatIntelAlert.getMonitorName()); + assertEquals(alert.getVersion(), newThreatIntelAlert.getVersion()); + assertEquals(alert.getActionExecutionResults(), newThreatIntelAlert.getActionExecutionResults()); + assertEquals(alert.getStartTime(), newThreatIntelAlert.getStartTime()); + assertEquals(alert.getAcknowledgedTime(), newThreatIntelAlert.getAcknowledgedTime()); + assertEquals(alert.getState(), newThreatIntelAlert.getState()); + assertEquals(alert.getIocValue(), newThreatIntelAlert.getIocValue()); + assertEquals(alert.getIocType(), newThreatIntelAlert.getIocType()); + assertEquals(alert.getLastUpdatedTime(), newThreatIntelAlert.getLastUpdatedTime()); + assertTrue(alert.getFindingIds().containsAll(newThreatIntelAlert.getFindingIds())); + } + + public void testThreatIntelAlertParse() throws IOException { + long now = System.currentTimeMillis(); + String threatIntelAlertString = "{\n" + + " \"id\": \"example-id\",\n" + + " \"version\": 1,\n" + + " \"schema_version\": 1,\n" + + " \"user\": null,\n" + + " \"trigger_name\": \"example-trigger-name\",\n" + + " \"trigger_id\": \"example-trigger-id\",\n" + + " \"monitor_id\": \"example-monitor-id\",\n" + + " \"monitor_name\": \"example-monitor-name\",\n" + + " \"state\": \"ACTIVE\",\n" + + " \"start_time\": \"" + now + "\",\n" + + " \"end_time\": \"" + now + "\",\n" + + " \"acknowledged_time\": \"" + now + "\",\n" + + " \"last_updated_time\": \"" + now + "\",\n" + + " \"ioc_value\": \"" + now + "\",\n" + + " \"ioc_type\": \"" + now + "\",\n" + + " \"error_message\": \"example-error-message\",\n" + + " \"severity\": \"high\",\n" + + " \"action_execution_results\": [],\n" + + " \"finding_id\": [ \"f1\", \"f2\"]\n" + + "}\n"; + + ThreatIntelAlert alert = ThreatIntelAlert.parse(getParser(threatIntelAlertString), 1l); + BytesStreamOutput out = new BytesStreamOutput(); + alert.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + ThreatIntelAlert newThreatIntelAlert = new ThreatIntelAlert(sin); + asserts(alert, newThreatIntelAlert); + } + + public XContentParser getParser(String xc) throws IOException { + XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); + parser.nextToken(); + return parser; + + } + + private static ThreatIntelAlert getRandomAlert() { + return new ThreatIntelAlert( + randomAlphaOfLength(10), + randomLong(), + randomLong(), + new User(randomAlphaOfLength(10), List.of(randomAlphaOfLength(10)), List.of(randomAlphaOfLength(10)), List.of(randomAlphaOfLength(10))), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + Alert.State.ACKNOWLEDGED, + Instant.now(), + Instant.now(), + Instant.now(), + Instant.now(), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + Collections.emptyList(), + List.of(randomAlphaOfLength(10), randomAlphaOfLength(10)) + ); + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index a156344b6..9ac49abbf 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -12,6 +12,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; +import org.opensearch.common.settings.Settings; import org.opensearch.commons.alerting.model.Monitor.MonitorType; import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; @@ -1042,7 +1043,7 @@ public void testCreateDetector_verifyWorkflowCreation_success_WithoutGroupByRule " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(2, response.getHits().getTotalHits().value); @@ -1110,7 +1111,7 @@ public void testCreateDetector_verifyWorkflowCreation_success_WithGroupByRulesIn " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(2, response.getHits().getTotalHits().value); @@ -1379,7 +1380,7 @@ public void testCreateDetector_workflowWithDuplicateMonitor_failure() throws IOE " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(2, response.getHits().getTotalHits().value); @@ -1449,7 +1450,7 @@ public void testCreateDetector_verifyWorkflowExecutionBucketLevelDocLevelMonitor " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(2, response.getHits().getTotalHits().value); @@ -2063,7 +2064,10 @@ public void testCreateDetectorWithCloudtrailAggrRuleWithDotFields() throws IOExc @SuppressWarnings("unchecked") public void testCreateDetectorWithCloudtrailAggrRuleWithEcsFields() throws IOException { - String index = createTestIndex("cloudtrail", cloudtrailOcsfMappings()); + String index = "cloudtrail"; + String indexAlias = "test_alias"; + + createIndex(index, Settings.EMPTY, cloudtrailOcsfMappings(), "\"" + indexAlias + "\":{\"is_write_index\": true}"); // Execute CreateMappingsAction to add alias mapping for index Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); @@ -2153,6 +2157,108 @@ public void testCreateDetectorWithCloudtrailAggrRuleWithEcsFields() throws IOExc assertEquals(1, getFindingsBody.get("total_findings")); } + @SuppressWarnings("unchecked") + public void testCreateDetectorWithCloudtrailAggrRuleWithRolloverIndexAliases() throws IOException, InterruptedException { + createSampleIndexTemplate("cloudtrail*", cloudtrailOcsfMappings(), true); + String index = createTestIndex("cloudtrail-000001", ""); + createIndexAliasApi("ocsf_ct", "cloudtrail-000001"); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{\n" + + " \"index_name\": \"ocsf_ct\",\n" + + " \"rule_topic\": \"cloudtrail\",\n" + + " \"partial\": true,\n" + + " \"alias_mappings\": {\n" + + " \"properties\": {\n" + + " \"timestamp\": {\n" + + " \"path\": \"time\",\n" + + " \"type\": \"alias\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + indexDoc(index, "0", randomCloudtrailOcsfDoc()); + + String rule = randomCloudtrailAggrRuleWithDotFields(); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("cloudtrail detector for security analytics", List.of("ocsf_ct"), List.of(new DetectorRule(createdId)), + List.of()); + Detector detector = randomDetectorWithInputsAndTriggers(List.of(input), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(createdId), List.of(), List.of(), List.of(), List.of()))); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + createdId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, createdId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, createdId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String workflowId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("workflow_ids")).get(0); + + indexDoc("ocsf_ct", "1", randomCloudtrailOcsfDoc()); + indexDoc("ocsf_ct", "2", randomCloudtrailOcsfDoc()); + executeAlertingWorkflow(workflowId, Collections.emptyMap()); + + Map params = new HashMap<>(); + params.put("detector_id", createdId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + // Assert findings + assertNotNull(getFindingsBody); + assertEquals(1, getFindingsBody.get("total_findings")); + + doRollover("ocsf_ct"); + Thread.sleep(90000); + + indexDoc("ocsf_ct", "4", randomCloudtrailOcsfDoc()); + indexDoc("ocsf_ct", "5", randomCloudtrailOcsfDoc()); + executeAlertingWorkflow(workflowId, Collections.emptyMap()); + + params = new HashMap<>(); + params.put("detector_id", createdId); + getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + getFindingsBody = entityAsMap(getFindingsResponse); + + // Assert findings + assertNotNull(getFindingsBody); + assertEquals(2, getFindingsBody.get("total_findings")); + } + private static void assertRuleMonitorFinding(Map executeResults, String ruleId, int expectedDocCount, List expectedTriggerResult) { List> buckets = ((List>) (((Map) ((Map) ((Map) ((List) ((Map) executeResults.get("input_results")).get("results")).get(0)).get("aggregations")).get("result_agg")).get("buckets"))); Integer docCount = buckets.stream().mapToInt(it -> (Integer) it.get("doc_count")).sum(); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java index dc821d304..4aaf4f35b 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java @@ -19,11 +19,10 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; -import org.opensearch.common.settings.Settings; import org.opensearch.client.ResponseException; +import org.opensearch.common.settings.Settings; import org.opensearch.commons.alerting.model.IntervalSchedule; import org.opensearch.commons.alerting.model.Monitor.MonitorType; -import org.opensearch.commons.alerting.model.ScheduledJob; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.search.SearchHit; @@ -33,17 +32,34 @@ import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; +import org.opensearch.securityanalytics.model.DetectorTrigger; import java.io.IOException; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; -import org.opensearch.securityanalytics.model.DetectorTrigger; -import static org.junit.Assert.assertNotNull; -import static org.opensearch.securityanalytics.TestHelpers.*; +import static org.opensearch.securityanalytics.TestHelpers.productIndexAvgAggRule; +import static org.opensearch.securityanalytics.TestHelpers.productIndexCountAggRule; +import static org.opensearch.securityanalytics.TestHelpers.productIndexMapping; +import static org.opensearch.securityanalytics.TestHelpers.randomDetector; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputs; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndTriggers; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithTriggers; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithTriggersAndScheduleAndEnabled; +import static org.opensearch.securityanalytics.TestHelpers.randomDoc; +import static org.opensearch.securityanalytics.TestHelpers.randomIndex; +import static org.opensearch.securityanalytics.TestHelpers.randomProductDocument; +import static org.opensearch.securityanalytics.TestHelpers.randomProductDocumentWithTime; +import static org.opensearch.securityanalytics.TestHelpers.randomRule; +import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE; public class DetectorRestApiIT extends SecurityAnalyticsRestTestCase { @@ -145,7 +161,7 @@ private void validateDetectorDeletion(final String detectorId) throws IOExceptio Assert.assertEquals(0, hits.size()); } - @SuppressWarnings("unchecked") + @SuppressWarnings("unchecked") public void testCreatingADetector() throws IOException { String index = createTestIndex(randomIndex(), windowsIndexMapping()); @@ -706,6 +722,12 @@ public void testCreatingADetectorWithAggregationRules() throws IOException { if(MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(secondMonitorType)){ bucketLevelMonitorId = secondMonitorId; } + String thirdMonitorId = monitorIds.get(2); + String thirdMonitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + thirdMonitorId))).get("monitor")).get("monitor_type"); + monitorTypes.add(thirdMonitorType); + if(MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(thirdMonitorType)){ + bucketLevelMonitorId = thirdMonitorId; + } Assert.assertTrue(Arrays.asList(MonitorType.BUCKET_LEVEL_MONITOR.getValue(), MonitorType.DOC_LEVEL_MONITOR.getValue()).containsAll(monitorTypes)); indexDoc(index, "1", randomProductDocument()); @@ -759,7 +781,13 @@ public void testAggRuleCount() throws IOException { Map detectorAsMap = (Map) hit.getSourceAsMap().get("detector"); - String bucketLevelMonitorId = ((List) (detectorAsMap).get("monitor_id")).get(1); + String bucketLevelMonitorId = ""; + Map monitorOpts = ((Map) (detectorAsMap).get("bucket_monitor_id_rule_id")); + for (Map.Entry monitorOpt: monitorOpts.entrySet()) { + if (!(monitorOpt.getKey().equals("-1") || monitorOpt.getKey().equals("chained_findings_monitor"))) { + bucketLevelMonitorId = monitorOpt.getValue().toString(); + } + } // condition: sel | count(*) by name > 2 indexDoc(index, "1", randomProductDocument()); indexDoc(index, "2", randomProductDocument()); @@ -824,7 +852,7 @@ public void testUpdateADetector() throws IOException { " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); Assert.assertEquals(5, response.getHits().getTotalHits().value); String rule = randomRule(); @@ -853,7 +881,7 @@ public void testUpdateADetector() throws IOException { " }\n" + " }\n" + "}"; - response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); Assert.assertEquals(6, response.getHits().getTotalHits().value); } @@ -1040,10 +1068,10 @@ public void testDeletingADetector_single_Monitor() throws IOException { Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); // both req params and req body are supported createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index + "\"," + - " \"rule_topic\":\"" + randomDetectorType() + "\", " + - " \"partial\":true" + - "}" + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" ); Response response = client().performRequest(createMappingRequest); @@ -1053,12 +1081,12 @@ public void testDeletingADetector_single_Monitor() throws IOException { String detectorId1 = createDetector(detector1); String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId1 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(Detector.DETECTORS_INDEX, request); SearchHit hit = hits.get(0); @@ -1080,12 +1108,12 @@ public void testDeletingADetector_single_Monitor() throws IOException { String detectorId2 = createDetector(detector2); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId2 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); hit = hits.get(0); @@ -1121,22 +1149,22 @@ public void testDeletingADetector_single_Monitor() throws IOException { Assert.assertFalse(doesIndexExist(String.format(Locale.ROOT, ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId1 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); Assert.assertEquals(0, hits.size()); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId2 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); Assert.assertEquals(0, hits.size()); } @@ -1150,10 +1178,10 @@ public void testDeletingADetector_single_Monitor_workflow_enabled() throws IOExc Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); // both req params and req body are supported createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index + "\"," + - " \"rule_topic\":\"" + randomDetectorType() + "\", " + - " \"partial\":true" + - "}" + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" ); Response response = client().performRequest(createMappingRequest); @@ -1163,12 +1191,12 @@ public void testDeletingADetector_single_Monitor_workflow_enabled() throws IOExc String detectorId1 = createDetector(detector1); String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId1 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(Detector.DETECTORS_INDEX, request); SearchHit hit = hits.get(0); @@ -1190,12 +1218,12 @@ public void testDeletingADetector_single_Monitor_workflow_enabled() throws IOExc String detectorId2 = createDetector(detector2); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId2 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); hit = hits.get(0); @@ -1231,26 +1259,27 @@ public void testDeletingADetector_single_Monitor_workflow_enabled() throws IOExc Assert.assertFalse(doesIndexExist(String.format(Locale.ROOT, ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId1 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); Assert.assertEquals(0, hits.size()); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId2 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); Assert.assertEquals(0, hits.size()); } + @SuppressWarnings("unchecked") public void testDeletingADetector_oneDetectorType_multiple_ruleTopicIndex() throws IOException { String index1 = "test_index_1"; createIndex(index1, Settings.EMPTY); @@ -1268,6 +1297,10 @@ public void testDeletingADetector_oneDetectorType_multiple_ruleTopicIndex() thro List.of(index1) ); String detectorId1 = createDetector(detector1); + Response response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), + new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId1 + "\"}}}"), new BasicHeader("Content-Type", "application/json")); + String ruleTopicIndex1 = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) + .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; // Create detector #2 of type test_windows Detector detector2 = randomDetectorWithTriggers( @@ -1277,29 +1310,19 @@ public void testDeletingADetector_oneDetectorType_multiple_ruleTopicIndex() thro ); String detectorId2 = createDetector(detector2); - - Assert.assertTrue(doesIndexExist(".opensearch-sap-test_windows-detectors-queries-000001")); - Assert.assertTrue(doesIndexExist(".opensearch-sap-test_windows-detectors-queries-000002")); - - // Check if both query indices have proper settings applied from index template - Map settings = getIndexSettingsAsMap(".opensearch-sap-test_windows-detectors-queries-000001"); - assertTrue(settings.containsKey("index.analysis.char_filter.rule_ws_filter.pattern")); - assertTrue(settings.containsKey("index.hidden")); - settings = getIndexSettingsAsMap(".opensearch-sap-test_windows-detectors-queries-000002"); - assertTrue(settings.containsKey("index.analysis.char_filter.rule_ws_filter.pattern")); - assertTrue(settings.containsKey("index.hidden")); + response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), + new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId2 + "\"}}}"), new BasicHeader("Content-Type", "application/json")); + String ruleTopicIndex2 = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) + .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; Response deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId1, Collections.emptyMap(), null); Assert.assertEquals("Delete detector failed", RestStatus.OK, restStatus(deleteResponse)); - // We deleted 1 detector, but 1 detector with same type exists, so we expect queryIndex to be present - Assert.assertFalse(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); - Assert.assertTrue(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-000002", "test_windows"))); deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId2, Collections.emptyMap(), null); Assert.assertEquals("Delete detector failed", RestStatus.OK, restStatus(deleteResponse)); // We deleted all detectors of type windows, so we expect that queryIndex is deleted - Assert.assertFalse(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); - Assert.assertFalse(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-000002", "test_windows"))); + Assert.assertFalse(doesIndexExist(ruleTopicIndex1)); + Assert.assertFalse(doesIndexExist(ruleTopicIndex2)); String request = "{\n" + " \"query\" : {\n" + @@ -1445,9 +1468,9 @@ public void testCreatingADetectorWithTimestampFieldAliasMapping_verifyTimeRangeI Request updateRequest = new Request("PUT", SecurityAnalyticsPlugin.MAPPER_BASE_URI); updateRequest.setJsonEntity(MediaTypeRegistry.JSON.contentBuilder().map(Map.of( - "index_name", index, - "field", "time", - "alias", "timestamp")) + "index_name", index, + "field", "time", + "alias", "timestamp")) .toString()); Response apiResponse = client().performRequest(updateRequest); assertEquals(HttpStatus.SC_OK, apiResponse.getStatusLine().getStatusCode()); @@ -1706,4 +1729,157 @@ public void testDetector_withAlias_endToEnd_success() throws IOException { List findings = (List) getFindingsBody.get("findings"); Assert.assertEquals(findings.size(), 1); } + + @SuppressWarnings("unchecked") + public void testCreatingDetectorWithDynamicQueryIndexDisabledAndThenEnabledToUpdate() throws IOException { + updateClusterSetting("plugins.security_analytics.enable_detectors_with_dedicated_query_indices", "false"); + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId1 = responseBody.get("_id").toString(); + + detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + String detectorId2 = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId1 = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; + hits = executeSearch(Detector.DETECTORS_INDEX, request); + hit = hits.get(0); + + String monitorId2 = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", randomDoc()); + + Response executeResponse = executeAlertingMonitor(monitorId1, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + executeResponse = executeAlertingMonitor(monitorId2, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + Assert.assertTrue(doesIndexExist(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "-000001")); + + updateClusterSetting("plugins.security_analytics.enable_detectors_with_dedicated_query_indices", "true"); + + Response updateResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId1, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Update detector failed", RestStatus.OK, restStatus(updateResponse)); + + indexDoc(index, "2", randomDoc()); + + executeResponse = executeAlertingMonitor(monitorId1, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + executeResponse = executeAlertingMonitor(monitorId2, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), + new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId1 + "\"}}}"), new BasicHeader("Content-Type", "application/json")); + String ruleTopicIndex1 = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) + .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; + Assert.assertTrue(doesIndexExist(ruleTopicIndex1)); + } + + @SuppressWarnings("unchecked") + public void testCreatingDetectorWithDynamicQueryIndexEnabledAndThenDisabled() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId1 = responseBody.get("_id").toString(); + + response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), + new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId1 + "\"}}}"), new BasicHeader("Content-Type", "application/json")); + String ruleTopicIndex1 = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) + .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; + Assert.assertTrue(doesIndexExist(ruleTopicIndex1)); + Assert.assertFalse(doesIndexExist(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "-000001")); + + updateClusterSetting("plugins.security_analytics.enable_detectors_with_dedicated_query_indices", "false"); + + Response updateResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId1, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Update detector failed", RestStatus.OK, restStatus(updateResponse)); + + response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), + new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId1 + "\"}}}"), new BasicHeader("Content-Type", "application/json")); + ruleTopicIndex1 = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) + .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; + Assert.assertTrue(doesIndexExist(ruleTopicIndex1)); + + detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + Assert.assertTrue(doesIndexExist(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "-000001")); + } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java new file mode 100644 index 000000000..bc86e11a1 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java @@ -0,0 +1,350 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.resthandler; + +import org.junit.Assert; +import org.opensearch.client.Response; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.TestHelpers; +import org.opensearch.securityanalytics.model.DetailedSTIX2IOCDto; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; +import org.opensearch.securityanalytics.model.threatintel.IocWithFeeds; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsActionResponse; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.model.STIX2IOC; +import org.opensearch.securityanalytics.model.STIX2IOCDto; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService; +import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.util.STIX2IOCGenerator; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class ListIOCsRestApiIT extends SecurityAnalyticsRestTestCase { + + public void testListIOCsWithNoFindingsIndex() throws IOException { + // Delete findings system indexes if they exist + try { + makeRequest(client(), "DELETE", IocFindingService.IOC_FINDING_INDEX_PATTERN_REGEXP, Collections.emptyMap(), null); + } catch (IndexNotFoundException indexNotFoundException) { + logger.info("No threat intel findings indexes to delete."); + } catch (Exception e) { + logger.error(e.getMessage()); + } + + // Create IOCs + String searchString = "test-list-iocs-no-findings-index"; + Map iocs = new HashMap<>(); + for (int i = 0; i < 100; i++) { + String iocId = searchString + "-" + i; + iocs.put( + iocId, + new STIX2IOCDto( + iocId, + iocId + "-name", + new IOCType(IOCType.IPV4_TYPE), + "ipv4value" + i, + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L + ) + ); + } + + // Creating source config + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + "test_list_ioc_" + searchString, + "STIX", + SourceConfigType.IOC_UPLOAD, + null, + null, + null, + new IocUploadSource(null, new ArrayList<>(iocs.values())), + null, + null, + null, + null, + null, + null, + null, + false, + List.of(IOCType.IPV4_TYPE), + true + ); + + // Create the IOC system indexes using IOC_UPLOAD config + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(response)); + + // Call ListIOCs API + Map params = Map.of( + "searchString", searchString, + "size", "10000" + ); + Response iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), params, null); + Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); + Map respMap = asMap(iocResponse); + + // Evaluate response + int totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); + assertEquals(iocs.size(), totalHits); + + List> iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); + assertEquals(iocs.size(), iocHits.size()); + + iocHits.forEach((hit) -> { + String iocId = (String) hit.get(STIX2IOC.ID_FIELD); + String iocName = (String) hit.get(STIX2IOC.NAME_FIELD); + String iocValue = (String) hit.get(STIX2IOC.VALUE_FIELD); + + STIX2IOCDto iocDto = iocs.get(iocId); + assertNotNull(iocDto); + + assertEquals(iocDto.getId(), iocId); + assertEquals(iocDto.getName(), iocName); + assertEquals(iocDto.getValue(), iocValue); + + int findingsNum = (int) hit.get(DetailedSTIX2IOCDto.NUM_FINDINGS_FIELD); + int expectedNumFindings = 0; + assertEquals(expectedNumFindings, findingsNum); + }); + } + + public void testListIOCsBySearchString() throws IOException { + String searchString = "test-search-string"; + List iocs = List.of( + // The 'name' field matches the searchString + new STIX2IOCDto( + "id1", + searchString, + new IOCType(IOCType.IPV4_TYPE), + "ipv4value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L + ), + // The 'value' field matches the searchString + new STIX2IOCDto( + "id2", + TestHelpers.randomLowerCaseString(), + new IOCType(IOCType.IPV4_TYPE), + searchString, + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L + ), + // No fields match the searchString + new STIX2IOCDto( + "id3", + "name", + new IOCType(IOCType.IPV4_TYPE), + "ipv4value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L + ) + ); + + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + "test_list_ioc_searchstring", + "STIX", + SourceConfigType.IOC_UPLOAD, + null, + null, + null, + new IocUploadSource(null, iocs), + null, + null, + null, + null, + null, + null, + null, + false, + List.of(IOCType.IPV4_TYPE), + true + ); + + // Create the IOC system indexes using IOC_UPLOAD config + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(response)); + + // Retrieve IOCs matching searchString + Response iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Map.of("searchString", searchString), null); + Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); + Map respMap = asMap(iocResponse); + + // Evaluate response + int totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); + assertEquals(2, totalHits); + + List> iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); + assertEquals(2, iocHits.size()); + + int nameMatch = (int) iocHits.stream().filter((hit) -> Objects.equals(hit.get(STIX2IOC.NAME_FIELD), searchString)).count(); + int valueMatch = (int) iocHits.stream().filter((hit) -> Objects.equals(hit.get(STIX2IOC.VALUE_FIELD), searchString)).count(); + assertEquals(1, nameMatch); + assertEquals(1, valueMatch); + } + + // TODO: Implement additional tests using various query param combinations + + public void testListIOCsNumFindings() throws Exception { + // Create IOCs + String searchString = "test-list-iocs-num-findings"; + List iocs = new ArrayList<>(); + Map> iocIdFindingsNum = new HashMap<>(); + for (int i = 0; i < 5; i++) { + String iocId = searchString + "-" + i; + iocs.add( + new STIX2IOCDto( + iocId, + iocId + "-name", + new IOCType(IOCType.IPV4_TYPE), + "ipv4value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L + ) + ); + + // Confirming the ListIOCs API can return a findings count greater than 10,000 by giving the first IOC 10,005 findings + int numFindings = i == 0 ? 10005 : randomInt(10); + List iocFindings = generateIOCMatches(numFindings, iocId); + + // Tracking the number of findings expected for each IOC + iocIdFindingsNum.put(iocId, iocFindings); + } + + // Creating source config + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + "test_list_ioc_" + searchString, + "STIX", + SourceConfigType.IOC_UPLOAD, + null, + null, + null, + new IocUploadSource(null, iocs), + null, + null, + null, + null, + null, + null, + null, + false, + List.of(IOCType.IPV4_TYPE), + true + ); + + // Create the IOC system indexes using IOC_UPLOAD config + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(response)); + + // Generate IOC matches + for (Map.Entry> entry : iocIdFindingsNum.entrySet()) { + ingestIOCMatches(entry.getValue()); + } + + // Call ListIOCs API + Response iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Map.of("searchString", searchString), null); + Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); + Map respMap = asMap(iocResponse); + + // Evaluate response + int totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); + assertEquals(iocs.size(), totalHits); + + List> iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); + assertEquals(iocs.size(), iocHits.size()); + + iocHits.forEach((hit) -> { + String iocId = (String) hit.get(STIX2IOC.ID_FIELD); + int findingsNum = (int) hit.get(DetailedSTIX2IOCDto.NUM_FINDINGS_FIELD); + int expectedNumFindings = iocIdFindingsNum.get(iocId).size(); + assertEquals(expectedNumFindings, findingsNum); + }); + } + + private List generateIOCMatches(int numMatches, String iocId) { + List iocFindings = new ArrayList<>(); + String monitorId = randomAlphaOfLength(10); + String monitorName = randomAlphaOfLength(10); + for (int i = 0; i < numMatches; i++) { + iocFindings.add(new IocFinding( + randomAlphaOfLength(10), + randomList(1, 10, () -> randomAlphaOfLength(10)),//docIds + randomList(1, 10, () -> new IocWithFeeds( + iocId, + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10)) + ), //feedIds + monitorId, + monitorName, + randomAlphaOfLength(10), + IOCType.IPV4_TYPE, + Instant.now(), + randomAlphaOfLength(10) + )); + } + return iocFindings; + } + + private void ingestIOCMatches(List iocFindings) throws IOException { + for (IocFinding iocFinding: iocFindings) { + makeRequest(client(), "POST", IocFindingService.IOC_FINDING_ALIAS_NAME + "/_doc?refresh", Map.of(), + toHttpEntity(iocFinding)); + } + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java index f22f70877..75e09f034 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java @@ -17,9 +17,11 @@ import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; +import org.opensearch.securityanalytics.model.DetectorTrigger; import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -433,7 +435,7 @@ public void testOCSFCloudtrailGetMappingsViewApi() throws IOException { Assert.assertEquals(18, props.size()); // Verify unmapped index fields List unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); - assertEquals(20, unmappedIndexFields.size()); + assertEquals(21, unmappedIndexFields.size()); // Verify unmapped field aliases List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); assertEquals(24, unmappedFieldAliases.size()); @@ -455,7 +457,8 @@ public void testOCSFCloudtrailGetMappingsViewApiWithCustomRule() throws IOExcept Assert.assertEquals(18, props.size()); // Verify unmapped index fields List unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); - assertEquals(20, unmappedIndexFields.size()); + + assertEquals(21, unmappedIndexFields.size()); // Verify unmapped field aliases List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); assertEquals(24, unmappedFieldAliases.size()); @@ -475,7 +478,7 @@ public void testOCSFCloudtrailGetMappingsViewApiWithCustomRule() throws IOExcept Assert.assertEquals(18, props2.size()); // Verify unmapped index fields List unmappedIndexFields2 = (List) respMap2.get("unmapped_index_fields"); - assertEquals(20, unmappedIndexFields2.size()); + assertEquals(21, unmappedIndexFields2.size()); // Verify unmapped field aliases List unmappedFieldAliases2 = (List) respMap2.get("unmapped_field_aliases"); assertEquals(24, unmappedFieldAliases2.size()); @@ -498,13 +501,13 @@ public void testOCSFVpcflowGetMappingsViewApi() throws IOException { Map respMap = responseAsMap(response); // Verify alias mappings Map props = (Map) respMap.get("properties"); - Assert.assertEquals(20, props.size()); + Assert.assertEquals(21, props.size()); // Verify unmapped index fields List unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); - assertEquals(26, unmappedIndexFields.size()); + assertEquals(25, unmappedIndexFields.size()); // Verify unmapped field aliases List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); - assertEquals(5, unmappedFieldAliases.size()); + assertEquals(4, unmappedFieldAliases.size()); } @SuppressWarnings("unchecked") @@ -520,13 +523,13 @@ public void testOCSFRoute53GetMappingsViewApi() throws IOException { Map respMap = responseAsMap(response); // Verify alias mappings Map props = (Map) respMap.get("properties"); - Assert.assertEquals(11, props.size()); + Assert.assertEquals(12, props.size()); // Verify unmapped index fields List unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); - assertEquals(28, unmappedIndexFields.size()); + assertEquals(27, unmappedIndexFields.size()); // Verify unmapped field aliases List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); - assertEquals(11, unmappedFieldAliases.size()); + assertEquals(12, unmappedFieldAliases.size()); } @SuppressWarnings("unchecked") @@ -586,15 +589,102 @@ public void testRawRoute53GetMappingsViewApi() throws IOException { Map respMap = responseAsMap(response); // Verify alias mappings Map props = (Map) respMap.get("properties"); - Assert.assertEquals(14, props.size()); + Assert.assertEquals(16, props.size()); // Verify unmapped index fields List unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); - assertEquals(6, unmappedIndexFields.size()); + assertEquals(4, unmappedIndexFields.size()); // Verify unmapped field aliases List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); assertEquals(8, unmappedFieldAliases.size()); } + public void testCloudtrailPrincipalIdAndArnFieldsGenerateFinding() throws IOException { + // create an index with OCSF1.1 fields actor.user.uid and actor.user.uid_alt + String indexName = "test_index"; + String index = createTestIndex(indexName, ocsf11ReducedCloudtrailMappings()); + + // create the cloudtrail mappings + createMappingsAPI(indexName, "cloudtrail"); + + // create the custom rule + String rule = ocsf11Rule(); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + String ruleId = responseBody.get("_id").toString(); + + // create the detector that uses only the custom rule + Detector detector = randomDetector( + "cloudtrail-detector", + "cloudtrail", + null, + List.of( + new DetectorInput( + "cloudtrail detector for security analytics", + List.of(indexName), + List.of(new DetectorRule(ruleId)), + List.of() + ) + ), + List.of( + new DetectorTrigger( + null, + "cloudtrail-trigger", + "1", + List.of("cloudtrail"), + List.of(ruleId), + List.of(), + List.of(), + List.of(), + List.of() + ) + ), + null, + true, + null, + null, + false + ); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + String detectorId = responseBody.get("_id").toString(); + + // get the underlying alerting monitor for the detector so we can manually execute it + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String detectorType = (String) ((Map) hit.getSourceAsMap().get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", "cloudtrail", detectorType.toLowerCase(Locale.ROOT)); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + // index a document that should trigger a finding + indexDoc(index, "1", ocsf11Doc()); + + // execute detector by executing its underlying monitor + executeAlertingMonitor(monitorId, Collections.emptyMap()); + + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + Assert.assertEquals(1, getFindingsBody.get("total_findings")); + } + private String rawCloudtrailDoc() { return "{\n" + " \"eventVersion\": \"1.03\",\n" + @@ -2810,4 +2900,63 @@ private String rawVpcFlowMappings() { " }\n" + " }"; } + + private String ocsf11ReducedCloudtrailMappings() { + return "\"properties\": {\n" + + " \"actor.user.uid_alt\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"actor.user.uid\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }"; + } + + private String ocsf11Rule() { + return "title: Cloudtrail Principal ID Rule\n" + + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a123\n" + + "description: A rule that checks specifically for the cloudtrail principal ID field\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " selection:\n" + + " aws.cloudtrail.user_identity.principalId: abc\n" + + " condition: selection\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; + } + + public String ocsf11Doc() { + return "{\n" + + "\"actor.user.uid_alt\":\"abc\",\n" + + "\"actor.user.uid\":\"def\"\n" + + "}"; + } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java index 6d7f4b550..9c258e0e8 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java @@ -110,14 +110,14 @@ public void testCreatingARule_custom_category() throws IOException { } catch (ResponseException e) { assertEquals(HttpStatus.SC_BAD_REQUEST, e.getResponse().getStatusLine().getStatusCode()); Assert.assertTrue( - e.getMessage().contains("Invalid rule category") + e.getMessage().contains("Invalid rule category") ); } } public void testCreatingAggregationRule() throws SigmaError, IOException { Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "windows"), - new StringEntity(countAggregationTestRule()), new BasicHeader("Content-Type", "application/json")); + new StringEntity(countAggregationTestRule()), new BasicHeader("Content-Type", "application/json")); Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); @@ -130,24 +130,24 @@ public void testCreatingAggregationRule() throws SigmaError, IOException { String index = Rule.CUSTOM_RULES_INDEX; String request = "{\n" + - " \"query\": {\n" + - " \"nested\": {\n" + - " \"path\": \"rule\",\n" + - " \"query\": {\n" + - " \"bool\": {\n" + - " \"must\": [\n" + - " { \"match\": {\"rule.category\": \"windows\"}}\n" + - " ]\n" + - " }\n" + - " }\n" + - " }\n" + - " }\n" + - "}"; + " \"query\": {\n" + + " \"nested\": {\n" + + " \"path\": \"rule\",\n" + + " \"query\": {\n" + + " \"bool\": {\n" + + " \"must\": [\n" + + " { \"match\": {\"rule.category\": \"windows\"}}\n" + + " ]\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(index, request); XContentParser xcp = XContentType.JSON.xContent() - .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, hits.get(0).getSourceAsString()); + .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, hits.get(0).getSourceAsString()); Rule result = Rule.docParse(xcp, null, null); Assert.assertEquals(1, result.getAggregationQueries().size()); @@ -728,7 +728,7 @@ public void testDeletingUsedRule() throws IOException { " }\n" + " }\n" + "}"; - List hits = executeSearch(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request); + List hits = executeSearch(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request); Assert.assertEquals(2, hits.size()); Response deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.RULE_BASE_URI + "/" + createdId, Collections.singletonMap("forced", "true"), null); @@ -741,7 +741,7 @@ public void testDeletingUsedRule() throws IOException { " }\n" + " }\n" + "}"; - hits = executeSearch(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request); + hits = executeSearch(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request); Assert.assertEquals(0, hits.size()); index = Rule.CUSTOM_RULES_INDEX; diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java new file mode 100644 index 000000000..e3460e561 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java @@ -0,0 +1,886 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.securityanalytics.resthandler; + +import com.google.common.collect.ImmutableList; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.search.SearchHit; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.TestHelpers; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.commons.utils.testUtils.S3ObjectGenerator; +import org.opensearch.securityanalytics.model.STIX2IOC; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; +import org.opensearch.securityanalytics.threatIntel.model.S3Source; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.model.Source; +import org.opensearch.securityanalytics.util.STIX2IOCGenerator; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.JOB_INDEX_NAME; +import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.getAllIocIndexPatternById; + +/** + * The following system parameters must be specified to successfully run these tests: + * + * tests.SATIFSourceConfigRestApiIT.bucketName - the name of the S3 bucket to use for the tests + * tests.SATIFSourceConfigRestApiIT.region - the AWS region of the S3 bucket + * tests.SATIFSourceConfigRestApiIT.roleArn - the IAM role ARN to assume when making S3 calls + * + * The local system must have sufficient credentials to write to S3, delete from S3, and assume the provided role. + * + * These tests are disabled by default as there is no default value for the tests.s3connector.bucket system property. This is + * intentional as the tests will fail when run without the proper setup, such as during CI workflows. + * + * Example command to manually run this class's ITs: + * ./gradlew ':integTest' --tests "org.opensearch.securityanalytics.resthandler.SATIFSourceConfigRestApiIT" \ + * -Dtests.SATIFSourceConfigRestApiIT.bucketName= \ + * -Dtests.SATIFSourceConfigRestApiIT.region= \ + * -Dtests.SATIFSourceConfigRestApiIT.roleArn= + * + * Optionally, the following system parameter can be supplied to PREVENT the tests from cleaning up the bucket objects. + * This could be helpful when troubleshooting failing tests by investigating the data generated during execution. + * By default, the bucket objects (not the bucket) will be cleaned up after the tests. + * To disable cleanup, add the following system parameter. + * -Dtests.SATIFSourceConfigRestApiIT.cleanup=false + */ +public class SATIFSourceConfigRestApiIT extends SecurityAnalyticsRestTestCase { + + private String bucketName; + private String objectKey; + private String region; + private String roleArn; + private Source source; + private S3Client s3Client; + private S3ObjectGenerator s3ObjectGenerator; + private STIX2IOCGenerator stix2IOCGenerator; + + /** + * Is reassigned in the initSource function. + * Will only be TRUE if 'bucketName', 'region', and 'roleArn' are supplied through system params. + * Disables tests when FALSE. + */ + private boolean canRunTests; + + /** + * List of invalid type patterns for easy test execution + */ + private final List invalidTypes = ImmutableList.of( + "ip", // "ip" is not currently a supported IOCType + "ipv4_addr" // Currently, the supported IOCTypes do not contain underscores + ); + + @Before + public void initSource() { + // Retrieve system parameters needed to run the tests. Only retrieve once + if (bucketName == null) { + bucketName = System.getProperty("tests.SATIFSourceConfigRestApiIT.bucketName"); + region = System.getProperty("tests.SATIFSourceConfigRestApiIT.region"); + roleArn = System.getProperty("tests.SATIFSourceConfigRestApiIT.roleArn"); + } + + // Confirm necessary system params are provided + canRunTests = bucketName != null && !bucketName.isBlank() && + region != null && !region.isBlank() && + roleArn != null && !roleArn.isBlank(); + + // Exit test setup if necessary system params are not provided + if (!canRunTests) { + logger.info(getClass().getName() + " tests disabled."); + System.out.println(getClass().getName() + " tests disabled."); + return; + } + + // Only create the s3Client once + if (s3Client == null) { + s3Client = S3Client.builder() + .region(Region.of(region)) + .build(); + s3ObjectGenerator = new S3ObjectGenerator(s3Client, bucketName); + } + + // Refresh source for each test + objectKey = TestHelpers.randomLowerCaseString(); + source = new S3Source(bucketName, objectKey, region, roleArn); + } + + @After + public void afterTest() { + // Exit test cleanup if necessary system params are not provided + if (!canRunTests) return; + + // Delete the bucket object unless cleanup is disabled + if (!Objects.equals(System.getProperty("tests.SATIFSourceConfigRestApiIT.cleanup"), "false")) { + DeleteObjectResponse response = s3Client.deleteObject( + DeleteObjectRequest.builder() + .bucket(bucketName) + .key(objectKey) + .build() + ); + + // Confirm bucket object was deleted successfully + assertTrue( + String.format("Failed to delete object with key %s in bucket %s", objectKey, bucketName), + response.sdkHttpResponse().isSuccessful() + ); + } + + // Close the client + s3Client.close(); + } + + public void testCreateSATIFSourceConfigAndVerifyJobRan() throws IOException, InterruptedException { + // Only run tests when required system params are provided + if (!canRunTests) return; + + // Generate test IOCs, and upload them to S3 to create the bucket object. Feed creation fails if the bucket object doesn't exist. + int numOfIOCs = 1; + stix2IOCGenerator = new STIX2IOCGenerator(List.of(new IOCType(IOCType.IPV4_TYPE))); + s3ObjectGenerator.write(numOfIOCs, objectKey, stix2IOCGenerator); + assertEquals("Incorrect number of test IOCs generated.", numOfIOCs, stix2IOCGenerator.getIocs().size()); + + // Create test feed + String feedName = "test_feed_name"; + String feedFormat = "STIX2"; + SourceConfigType sourceConfigType = SourceConfigType.S3_CUSTOM; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES); + List iocTypes = List.of(IOCType.IPV4_TYPE, IOCType.DOMAIN_NAME_TYPE); + + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + Instant.now(), + source, + null, + Instant.now(), + schedule, + null, + null, + Instant.now(), + null, + true, + iocTypes, + true + ); + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, createdId), response.getHeader("Location")); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(1, hits.size()); + + // call get API to get the latest source config by ID + response = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/" + createdId, Collections.emptyMap(), null); + responseBody = asMap(response); + String firstUpdatedTime = (String) ((Map)responseBody.get("source_config")).get("last_update_time"); + + // wait for job runner to run + waitUntil(() -> { + try { + return verifyJobRan(createdId, firstUpdatedTime); + } catch (IOException e) { + throw new RuntimeException("failed to verify that job ran"); + } + }, 240, TimeUnit.SECONDS); + } + + public void testGetSATIFSourceConfigById() throws IOException { + // Only run tests when required system params are provided + if (!canRunTests) return; + + // Generate test IOCs, and upload them to S3 to create the bucket object. Feed creation fails if the bucket object doesn't exist. + int numOfIOCs = 1; + stix2IOCGenerator = new STIX2IOCGenerator(List.of(new IOCType(IOCType.HASHES_TYPE))); + s3ObjectGenerator.write(numOfIOCs, objectKey, stix2IOCGenerator); + assertEquals("Incorrect number of test IOCs generated.", numOfIOCs, stix2IOCGenerator.getIocs().size()); + + // Create test feed + String feedName = "test_feed_name"; + String feedFormat = "STIX2"; + SourceConfigType sourceConfigType = SourceConfigType.S3_CUSTOM; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); + List iocTypes = List.of(IOCType.HASHES_TYPE); + + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + Instant.now(), + source, + null, + Instant.now(), + schedule, + null, + null, + Instant.now(), + null, + true, + iocTypes, + true + ); + + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + response = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/" + createdId, Collections.emptyMap(), null); + responseBody = asMap(response); + + String responseId = responseBody.get("_id").toString(); + Assert.assertEquals("Created Id and returned Id do not match", createdId, responseId); + + int responseVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertTrue("Incorrect version", responseVersion > 0); + + String returnedFeedName = (String) ((Map)responseBody.get("source_config")).get("name"); + Assert.assertEquals("Created feed name and returned feed name do not match", feedName, returnedFeedName); + + String returnedFeedFormat = (String) ((Map)responseBody.get("source_config")).get("format"); + Assert.assertEquals("Created feed format and returned feed format do not match", feedFormat, returnedFeedFormat); + + String returnedFeedType = (String) ((Map)responseBody.get("source_config")).get("type"); + Assert.assertEquals("Created feed type and returned feed type do not match", sourceConfigType, SATIFSourceConfigDto.toSourceConfigType(returnedFeedType)); + + List returnedIocTypes = (List) ((Map)responseBody.get("source_config")).get("ioc_types"); + Assert.assertTrue("Created ioc types and returned ioc types do not match", iocTypes.containsAll(returnedIocTypes) && returnedIocTypes.containsAll(iocTypes)); + } + + public void testDeleteSATIFSourceConfig() throws IOException { + // Only run tests when required system params are provided + if (!canRunTests) return; + + // Generate test IOCs, and upload them to S3 to create the bucket object. Feed creation fails if the bucket object doesn't exist. + int numOfIOCs = 1; + stix2IOCGenerator = new STIX2IOCGenerator(List.of(new IOCType(IOCType.IPV4_TYPE))); + s3ObjectGenerator.write(numOfIOCs, objectKey, stix2IOCGenerator); + assertEquals("Incorrect number of test IOCs generated.", numOfIOCs, stix2IOCGenerator.getIocs().size()); + + // Create test feed + String feedName = "test_feed_name"; + String feedFormat = "STIX2"; + SourceConfigType sourceConfigType = SourceConfigType.S3_CUSTOM; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES); + List iocTypes = List.of(IOCType.IPV4_TYPE, IOCType.HASHES_TYPE); + + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + Instant.now(), + source, + null, + Instant.now(), + schedule, + null, + null, + Instant.now(), + null, + true, + iocTypes, + true + ); + + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, createdId), response.getHeader("Location")); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(1, hits.size()); + + // call delete API to delete the threat intel source config + response = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/" + createdId, Collections.emptyMap(), null); + Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + responseBody = asMap(response); + + String deletedId = responseBody.get("_id").toString(); + Assert.assertEquals(deletedId, createdId); + + hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(0, hits.size()); + } + + public void testRetrieveIOCsSuccessfully() throws IOException, InterruptedException { + // Only run tests when required system params are provided + if (!canRunTests) return; + + // Execute test for each IOCType + for (String type : IOCType.types) { + // Generate test IOCs, and upload them to S3 + int numOfIOCs = 5; + stix2IOCGenerator = new STIX2IOCGenerator(List.of(new IOCType(type))); + s3ObjectGenerator.write(numOfIOCs, objectKey, stix2IOCGenerator); + assertEquals("Incorrect number of test IOCs generated for type: " + type, numOfIOCs, stix2IOCGenerator.getIocs().size()); + + // Create test feed + String feedName = "download_test_feed_name"; + String feedFormat = "STIX2"; + SourceConfigType sourceConfigType = SourceConfigType.S3_CUSTOM; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES); + List iocTypes = List.of(type); + + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + Instant.now(), + source, + null, + Instant.now(), + schedule, + null, + null, + Instant.now(), + null, + true, + iocTypes, + true + ); + + // Confirm test feed was created successfully + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("Response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + + // Wait for feed to execute + String firstUpdatedTime = (String) ((Map)responseBody.get("source_config")).get("last_refreshed_time"); + waitUntil(() -> { + try { + return verifyJobRan(createdId, firstUpdatedTime); + } catch (IOException e) { + throw new RuntimeException("failed to verify that job ran"); + } + }, 240, TimeUnit.SECONDS); + + // Confirm IOCs were ingested to system index for the feed + String indexName = getAllIocIndexPatternById(createdId); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(indexName, request); + + // Confirm expected number of results are returned + assertEquals(numOfIOCs, hits.size()); + List> iocs = hits.stream() + .map(SearchHit::getSourceAsMap) + .collect(Collectors.toList()); + + // Sort IOC lists for easy comparison + stix2IOCGenerator.getIocs().sort(Comparator.comparing(STIX2IOC::getName)); + iocs.sort(Comparator.comparing(ioc -> (String) ioc.get(STIX2IOC.NAME_FIELD))); + + // Confirm expected IOCs have been ingested + for (int i = 0; i < numOfIOCs; i++) { + assertEquals(stix2IOCGenerator.getIocs().get(i).getName(), iocs.get(i).get(STIX2IOC.NAME_FIELD)); + assertEquals(stix2IOCGenerator.getIocs().get(i).getType().toString(), IOCType.fromString((String) iocs.get(i).get(STIX2IOC.TYPE_FIELD))); + assertEquals(stix2IOCGenerator.getIocs().get(i).getValue(), iocs.get(i).get(STIX2IOC.VALUE_FIELD)); + assertEquals(stix2IOCGenerator.getIocs().get(i).getSeverity(), iocs.get(i).get(STIX2IOC.SEVERITY_FIELD)); + + // TODO troubleshoot instant assertions +// assertEquals(stix2IOCGenerator.getIocs().get(i).getCreated().toString(), iocs.get(i).get(STIX2IOC.CREATED_FIELD)); +// assertEquals(stix2IOCGenerator.getIocs().get(i).getModified().toString(), iocs.get(i).get(STIX2IOC.MODIFIED_FIELD)); + + assertEquals(stix2IOCGenerator.getIocs().get(i).getDescription(), iocs.get(i).get(STIX2IOC.DESCRIPTION_FIELD)); + assertEquals(stix2IOCGenerator.getIocs().get(i).getLabels(), iocs.get(i).get(STIX2IOC.LABELS_FIELD)); + assertEquals(createdId, iocs.get(i).get(STIX2IOC.FEED_ID_FIELD)); + assertEquals(stix2IOCGenerator.getIocs().get(i).getSpecVersion(), iocs.get(i).get(STIX2IOC.SPEC_VERSION_FIELD)); + } + } + } + + public void testRetrieveMultipleIOCTypesSuccessfully() throws IOException, InterruptedException { + // Only run tests when required system params are provided + if (!canRunTests) return; + + List numOfIOCsList = List.of( + 5, + 1000001 // Over 1 million IOCs + ); + + for (int numOfIOCs : numOfIOCsList) { + // Generate test IOCs for each type, and upload them to S3 + stix2IOCGenerator = new STIX2IOCGenerator(); + s3ObjectGenerator.write(numOfIOCs, objectKey, stix2IOCGenerator); + List allIocs = stix2IOCGenerator.getIocs(); + assertEquals("Incorrect total number of test IOCs generated.", IOCType.types.size() * numOfIOCs, allIocs.size()); + + // Create test feed + String feedName = "download_test_feed_name"; + String feedFormat = "STIX2"; + SourceConfigType sourceConfigType = SourceConfigType.S3_CUSTOM; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES); + + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + Instant.now(), + source, + null, + Instant.now(), + schedule, + null, + null, + Instant.now(), + null, + true, + IOCType.types, + true + ); + + // Confirm test feed was created successfully + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("Response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + + // Wait for feed to execute + String firstUpdatedTime = (String) ((Map)responseBody.get("source_config")).get("last_refreshed_time"); + waitUntil(() -> { + try { + return verifyJobRan(createdId, firstUpdatedTime); + } catch (IOException e) { + throw new RuntimeException("failed to verify that job ran"); + } + }, 240, TimeUnit.SECONDS); + + // Confirm IOCs were ingested to system index for the feed + String indexName = getAllIocIndexPatternById(createdId); + + String request = "{\n" + + " \"size\" : 10000,\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(indexName, request); + + // Confirm expected number of results are returned + assertEquals(allIocs.size(), hits.size()); + List> iocHits = hits.stream() + .map(SearchHit::getSourceAsMap) + .collect(Collectors.toList()); + + // Sort IOC lists for easy comparison + allIocs.sort(Comparator.comparing(STIX2IOC::getName)); + iocHits.sort(Comparator.comparing(ioc -> (String) ioc.get(STIX2IOC.NAME_FIELD))); + + // Confirm expected IOCs have been ingested + for (int i = 0; i < allIocs.size(); i++) { + assertEquals(stix2IOCGenerator.getIocs().get(i).getName(), iocHits.get(i).get(STIX2IOC.NAME_FIELD)); + assertEquals(stix2IOCGenerator.getIocs().get(i).getType().toString(), IOCType.fromString((String) iocHits.get(i).get(STIX2IOC.TYPE_FIELD))); + assertEquals(stix2IOCGenerator.getIocs().get(i).getValue(), iocHits.get(i).get(STIX2IOC.VALUE_FIELD)); + assertEquals(stix2IOCGenerator.getIocs().get(i).getSeverity(), iocHits.get(i).get(STIX2IOC.SEVERITY_FIELD)); + + // TODO troubleshoot instant assertions +// assertEquals(stix2IOCGenerator.getIocs().get(i).getCreated().toString(), iocHits.get(i).get(STIX2IOC.CREATED_FIELD)); +// assertEquals(stix2IOCGenerator.getIocs().get(i).getModified().toString(), iocHits.get(i).get(STIX2IOC.MODIFIED_FIELD)); + + assertEquals(stix2IOCGenerator.getIocs().get(i).getDescription(), iocHits.get(i).get(STIX2IOC.DESCRIPTION_FIELD)); + assertEquals(stix2IOCGenerator.getIocs().get(i).getLabels(), iocHits.get(i).get(STIX2IOC.LABELS_FIELD)); + assertEquals(createdId, iocHits.get(i).get(STIX2IOC.FEED_ID_FIELD)); + assertEquals(stix2IOCGenerator.getIocs().get(i).getSpecVersion(), iocHits.get(i).get(STIX2IOC.SPEC_VERSION_FIELD)); + } + } + } + + public void testWithValidAndInvalidIOCTypes() throws IOException { + // Only run tests when required system params are provided + if (!canRunTests) return; + + // Generate test IOCs, and upload them to S3 + int numOfIOCs = 5; + stix2IOCGenerator = new STIX2IOCGenerator(List.of(new IOCType(IOCType.IPV4_TYPE))); + s3ObjectGenerator.write(numOfIOCs, objectKey, stix2IOCGenerator); + assertEquals("Incorrect number of test IOCs generated.", numOfIOCs, stix2IOCGenerator.getIocs().size()); + + List types = new ArrayList<>(invalidTypes); + types.addAll(IOCType.types); + + // Execute the test for each invalid type + for (String type : invalidTypes) { + // Create test feed + String feedName = "download_test_feed_name"; + String feedFormat = "STIX2"; + SourceConfigType sourceConfigType = SourceConfigType.S3_CUSTOM; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES); + + List iocTypes = List.of(type); + + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + Instant.now(), + source, + null, + Instant.now(), + schedule, + null, + null, + Instant.now(), + null, + true, + iocTypes, + true + ); + + Exception exception = assertThrows(ResponseException.class, () -> + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)) + ); + + String expectedError = "{\"error\":{\"root_cause\":[{\"type\":\"status_exception\",\"reason\":\"No compatible Iocs were downloaded for config download_test_feed_name\"}],\"type\":\"status_exception\",\"reason\":\"No compatible Iocs were downloaded for config download_test_feed_name\"},\"status\":400}"; + assertTrue(exception.getMessage().contains(expectedError)); + } + } + + public void testWithInvalidIOCTypes() throws IOException { + // Only run tests when required system params are provided + if (!canRunTests) return; + + // Generate test IOCs, and upload them to S3 + int numOfIOCs = 5; + stix2IOCGenerator = new STIX2IOCGenerator(List.of(new IOCType(IOCType.IPV4_TYPE))); + s3ObjectGenerator.write(numOfIOCs, objectKey, stix2IOCGenerator); + assertEquals("Incorrect number of test IOCs generated.", numOfIOCs, stix2IOCGenerator.getIocs().size()); + + // Execute the test for each invalid type + for (String type : invalidTypes) { + // Create test feed + String feedName = "download_test_feed_name"; + String feedFormat = "STIX2"; + SourceConfigType sourceConfigType = SourceConfigType.S3_CUSTOM; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES); + + List iocTypes = List.of(type); + + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + Instant.now(), + source, + null, + Instant.now(), + schedule, + null, + null, + Instant.now(), + null, + true, + iocTypes, + true + ); + + Exception exception = assertThrows(ResponseException.class, () -> + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)) + ); + + String expectedError = "{\"error\":{\"root_cause\":[{\"type\":\"status_exception\",\"reason\":\"No compatible Iocs were downloaded for config download_test_feed_name\"}],\"type\":\"status_exception\",\"reason\":\"No compatible Iocs were downloaded for config download_test_feed_name\"},\"status\":400}"; + assertTrue(exception.getMessage().contains(expectedError)); + } + } + + public void testWithNoIOCsToDownload() { + // Only run tests when required system params are provided + if (!canRunTests) return; + + // Create the bucket object without any IOCs + PutObjectRequest putObjectRequest = PutObjectRequest.builder() + .bucket(bucketName) + .key(objectKey) + .build(); + PutObjectResponse putObjectResponse = s3Client.putObject(putObjectRequest, RequestBody.empty()); + assertTrue("Failed to create empty bucket object for type.", putObjectResponse.sdkHttpResponse().isSuccessful()); + + // Execute the test case for each IOC type + for (String type : IOCType.types) { + // Create test feed + String feedName = "download_test_feed_name"; + String feedFormat = "STIX2"; + SourceConfigType sourceConfigType = SourceConfigType.S3_CUSTOM; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES); + List iocTypes = List.of(type); + + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + Instant.now(), + source, + null, + Instant.now(), + schedule, + null, + null, + Instant.now(), + null, + true, + iocTypes, + true + ); + + Exception exception = assertThrows(ResponseException.class, () -> + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)) + ); + + String expectedError = "{\"error\":{\"root_cause\":[{\"type\":\"status_exception\",\"reason\":\"No compatible Iocs were downloaded for config download_test_feed_name\"}],\"type\":\"status_exception\",\"reason\":\"No compatible Iocs were downloaded for config download_test_feed_name\"},\"status\":400}"; + assertTrue(exception.getMessage().contains(expectedError)); + } + } + + public void testWhenBucketObjectDoesNotExist() throws IOException { + // Only run tests when required system params are provided + if (!canRunTests) return; + + // Confirm bucket object does not exist + HeadObjectRequest headObjectRequest = HeadObjectRequest.builder() + .bucket(bucketName) + .key(objectKey) + .build(); + assertThrows( + String.format("Object %s in bucket %s should not exist.", objectKey, bucketName), + NoSuchKeyException.class, () -> s3Client.headObject(headObjectRequest) + ); + + // Execute the test case for each IOC type + for (String type : IOCType.types) { + // Create test feed + String feedName = "download_test_feed_name"; + String feedFormat = "STIX2"; + SourceConfigType sourceConfigType = SourceConfigType.S3_CUSTOM; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES); + List iocTypes = List.of(type); + + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + Instant.now(), + source, + null, + Instant.now(), + schedule, + null, + null, + Instant.now(), + null, + true, + iocTypes, + true + ); + + try { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + } catch (ResponseException exception) { + assertEquals(RestStatus.NOT_FOUND, restStatus(exception.getResponse())); + String expectedError = "The specified key does not exist."; + assertTrue("Exception contains unexpected message: " + exception.getMessage(), exception.getMessage().contains(expectedError)); + } + } + + // ensure that source config was deleted + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(0, hits.size()); + + // ensure that ioc indices were deleted + request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(0, hits.size()); + } + + public void testWhenRoleArnIsEmpty() throws IOException { + // Try to create a source config with empty roleArn + source = new S3Source("bucketName", "objectKey", "region", ""); + + // Create test feed + String feedName = "download_test_feed_name"; + String feedFormat = "STIX2"; + SourceConfigType sourceConfigType = SourceConfigType.S3_CUSTOM; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES); + List iocTypes = List.of(IOCType.IPV4_TYPE); + + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + Instant.now(), + source, + null, + Instant.now(), + schedule, + null, + null, + Instant.now(), + null, + true, + iocTypes, + true + ); + + Exception exception = assertThrows(ResponseException.class, () -> + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)) + ); + + String expectedError = "Role arn is empty or malformed"; + assertTrue("Exception contains unexpected message: " + exception.getMessage(), exception.getMessage().contains(expectedError)); + + // ensure that source config is not created + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(0, hits.size()); + } + + /** + * Calls the get source config api and checks if the last updated time is different from the time that was passed in + * @param createdId + * @param firstUpdatedTime + * @return + * @throws IOException + */ + protected boolean verifyJobRan(String createdId, String firstUpdatedTime) throws IOException { + Response response; + Map responseBody; + + // call get API to get the latest source config by ID + response = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/" + createdId, Collections.emptyMap(), null); + responseBody = asMap(response); + + String returnedLastUpdatedTime = (String) ((Map) responseBody.get("source_config")).get("last_update_time"); + + if(firstUpdatedTime.equals(returnedLastUpdatedTime.toString()) == false) { + return true; + } + return false; + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java new file mode 100644 index 000000000..9b457f7df --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java @@ -0,0 +1,1061 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.securityanalytics.resthandler; + +import org.apache.http.entity.StringEntity; +import org.apache.http.message.BasicHeader; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.Assert; +import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; +import org.opensearch.client.WarningFailureException; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.search.SearchHit; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsActionRequest; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsActionResponse; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.model.STIX2IOCDto; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; +import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; +import org.opensearch.securityanalytics.util.STIX2IOCGenerator; + +import java.io.IOException; +import java.net.URL; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.opensearch.jobscheduler.spi.utils.LockService.LOCK_INDEX_NAME; +import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.JOB_INDEX_NAME; +import static org.opensearch.securityanalytics.TestHelpers.oldThreatIntelJobMapping; +import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.IOC_ALL_INDEX_PATTERN; +import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.getAllIocIndexPatternById; + +public class SourceConfigWithoutS3RestApiIT extends SecurityAnalyticsRestTestCase { + private static final Logger log = LogManager.getLogger(SourceConfigWithoutS3RestApiIT.class); + + public void testCreateIocUploadSourceConfig() throws IOException { + String feedName = "test_ioc_upload"; + String feedFormat = "STIX"; + SourceConfigType sourceConfigType = SourceConfigType.IOC_UPLOAD; + + List iocs = List.of(new STIX2IOCDto( + "id", + "name", + new IOCType(IOCType.IPV4_TYPE), + "value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L)); + + IocUploadSource iocUploadSource = new IocUploadSource(null, iocs); + Boolean enabled = false; + List iocTypes = List.of(IOCType.IPV4_TYPE); + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + iocUploadSource, + null, + null, + null, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(response)); + Map responseBody = asMap(response); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, createdId), response.getHeader("Location")); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(1, hits.size()); + + // ensure same number of iocs got indexed + String indexName = getAllIocIndexPatternById(createdId); + hits = executeSearch(indexName, request); + Assert.assertEquals(iocs.size(), hits.size()); + + // Retrieve all IOCs + Response iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Collections.emptyMap(), null); + Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); + Map respMap = asMap(iocResponse); + + // Evaluate response + int totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); + assertTrue(iocs.size() < totalHits); //due to default feed leading to more iocs + + List> iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); + assertTrue(iocs.size() < iocHits.size()); + // Retrieve all IOCs by feed Ids + iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Map.of("feed_ids", createdId + ",random"), null); + Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); + respMap = asMap(iocResponse); + + // Evaluate response + totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); + assertEquals(iocs.size(), totalHits); + + iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); + assertEquals(iocs.size(), iocHits.size()); + // Retrieve all IOCs by ip types + Map params = Map.of( + ListIOCsActionRequest.TYPE_FIELD, + String.format("%s,%s", IOCType.IPV4_TYPE, IOCType.DOMAIN_NAME_TYPE) + ); + iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), params, null); + Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); + respMap = asMap(iocResponse); + + // Evaluate response + totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); + assertTrue(iocs.size() < totalHits); + + iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); + assertTrue(iocs.size() < iocHits.size()); + } + + public void testCreateIocUploadSourceConfigIncorrectIocTypes() throws IOException { + // Attempt to create ioc upload source config with no correct ioc types + String feedName = "test_ioc_upload"; + String feedFormat = "STIX"; + SourceConfigType sourceConfigType = SourceConfigType.IOC_UPLOAD; + + List iocs = List.of(new STIX2IOCDto( + "id", + "name", + new IOCType(IOCType.IPV4_TYPE), + "value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L)); + + IocUploadSource iocUploadSource = new IocUploadSource(null, iocs); + Boolean enabled = false; + List iocTypes = List.of(IOCType.DOMAIN_NAME_TYPE); + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + iocUploadSource, + null, + null, + null, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + + try { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + } catch (ResponseException ex) { + Assert.assertEquals(RestStatus.BAD_REQUEST, restStatus(ex.getResponse())); + } + } + + public void testUpdateIocUploadSourceConfig() throws IOException { + // Create source config with IPV4 IOCs + String feedName = "test_update"; + String feedFormat = "STIX"; + SourceConfigType sourceConfigType = SourceConfigType.IOC_UPLOAD; + + List iocs = List.of(new STIX2IOCDto( + "1", + "ioc", + new IOCType(IOCType.IPV4_TYPE), + "value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L)); + + IocUploadSource iocUploadSource = new IocUploadSource(null, iocs); + Boolean enabled = false; + List iocTypes = List.of("ipv4-addr"); + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + iocUploadSource, + null, + null, + null, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + + // create source config with ipv4 ioc type + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(response)); + Map responseBody = asMap(response); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, createdId), response.getHeader("Location")); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(1, hits.size()); + + // ensure same number of iocs got indexed + String indexName = getAllIocIndexPatternById(createdId); + hits = executeSearch(indexName, request); + Assert.assertEquals(iocs.size(), hits.size()); + + // Retrieve all IOCs by feed Ids + Response iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Map.of("feed_ids", createdId + ",random"), null); + Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); + Map respMap = asMap(iocResponse); + + // Evaluate response + int totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); + assertEquals(iocs.size(), totalHits); + + List> iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); + assertEquals(iocs.size(), iocHits.size()); + + // update source config to contain only hashes as an ioc type + iocs = List.of(new STIX2IOCDto( + "2", + "ioc", + new IOCType(IOCType.HASHES_TYPE), + "value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L), + new STIX2IOCDto( + "3", + "ioc", + new IOCType(IOCType.DOMAIN_NAME_TYPE), + "value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L)); + + iocUploadSource = new IocUploadSource(null, iocs); + iocTypes = List.of("hashes"); + saTifSourceConfigDto = new SATIFSourceConfigDto( + saTifSourceConfigDto.getId(), + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + iocUploadSource, + null, + null, + null, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + + // update source config with hashes ioc type + response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + createdId, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.OK, restStatus(response)); + + // Ensure that old ioc indices are retained (2 created from ioc upload source config + 1 from default source config) + List findingIndices = getIocIndices(); + Assert.assertEquals(3, findingIndices.size()); + + // Retrieve all IOCs by feed Ids + iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Map.of("feed_ids", createdId + ",random"), null); + Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); + respMap = asMap(iocResponse); + + // Evaluate response - there should only be 1 ioc indexed according to the ioc type + totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); + assertEquals(1, totalHits); + + iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); + assertEquals(1, iocHits.size()); + } + + public void testActivateDeactivateIocUploadSourceConfig() throws IOException { + // Create source config with IPV4 IOCs + String feedName = "test_update"; + String feedFormat = "STIX"; + SourceConfigType sourceConfigType = SourceConfigType.IOC_UPLOAD; + + List iocs = List.of(new STIX2IOCDto( + "1", + "ioc", + new IOCType(IOCType.IPV4_TYPE), + "value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L)); + + IocUploadSource iocUploadSource = new IocUploadSource(null, iocs); + Boolean enabled = false; + List iocTypes = List.of("ipv4-addr"); + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + iocUploadSource, + null, + null, + null, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + + // create source config with ipv4 ioc type + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(response)); + Map responseBody = asMap(response); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, createdId), response.getHeader("Location")); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(1, hits.size()); + + // ensure same number of iocs got indexed + String indexName = getAllIocIndexPatternById(createdId); + hits = executeSearch(indexName, request); + Assert.assertEquals(iocs.size(), hits.size()); + + // Retrieve all IOCs by feed Ids + Response iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Map.of("feed_ids", createdId + ",random"), null); + Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); + Map respMap = asMap(iocResponse); + + // Evaluate response + int totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); + assertEquals(iocs.size(), totalHits); + + List> iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); + assertEquals(iocs.size(), iocHits.size()); + + // update source config to contain only hashes as an ioc type + iocs = Collections.emptyList(); + + iocUploadSource = new IocUploadSource(null, iocs); + iocTypes = List.of("hashes"); + saTifSourceConfigDto = new SATIFSourceConfigDto( + saTifSourceConfigDto.getId(), + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + iocUploadSource, + null, + null, + null, + null, + null, + null, + null, + enabled, + iocTypes, false + ); + + // update source config with hashes ioc type + response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + createdId, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.OK, restStatus(response)); + Map updateResponseAsMap = asMap(response); + assertNotNull(updateResponseAsMap); + assertTrue(updateResponseAsMap.containsKey("source_config")); + HashMap scr = (HashMap) updateResponseAsMap.get("source_config"); + assertTrue(scr.containsKey("enabled")); + assertFalse((Boolean) scr.get("enabled")); + assertTrue(scr.containsKey("enabled_for_scan")); + assertFalse((Boolean) scr.get("enabled_for_scan")); + + // Ensure that old ioc indices are retained (2 created from ioc upload source config + 1 from default source config) + List findingIndices = getIocIndices(); + Assert.assertEquals(2, findingIndices.size()); + + // Retrieve all IOCs by feed Ids + iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Map.of("feed_ids", createdId + ",random"), null); + Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); + respMap = asMap(iocResponse); + + // Evaluate response - there should only be 1 ioc indexed according to the ioc type + totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); + assertEquals(1, totalHits); + + iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); + assertEquals(1, iocHits.size()); + + saTifSourceConfigDto = new SATIFSourceConfigDto( + saTifSourceConfigDto.getId(), + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + iocUploadSource, + null, + null, + null, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + + // update source config with hashes ioc type + response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + createdId, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.OK, restStatus(response)); + updateResponseAsMap = asMap(response); + assertNotNull(updateResponseAsMap); + assertTrue(updateResponseAsMap.containsKey("source_config")); + scr = (HashMap) updateResponseAsMap.get("source_config"); + assertTrue(scr.containsKey("enabled")); + assertFalse((Boolean) scr.get("enabled")); // since its not url_download type, this flag should remain unaffected by the activate action in update source api + assertTrue(scr.containsKey("enabled_for_scan")); + assertTrue((Boolean) scr.get("enabled_for_scan")); + } + + public void testActivateDeactivateUrlDownloadSourceConfig() throws IOException { + // Search source configs when none are created + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + + // Search all source configs + Response sourceConfigResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/_search", Collections.emptyMap(), new StringEntity(request), new BasicHeader("Content-type", "application/json")); + Assert.assertEquals(RestStatus.OK, restStatus(sourceConfigResponse)); + Map responseBody = asMap(sourceConfigResponse); + + // Expected value is 1 - only default source config + Assert.assertEquals(1, ((Map) ((Map) responseBody.get("hits")).get("total")).get("value")); + + // Update default source config + String feedName = "test_update_default"; + String feedFormat = "STIX"; + SourceConfigType sourceConfigType = SourceConfigType.URL_DOWNLOAD; + UrlDownloadSource urlDownloadSource = new UrlDownloadSource(new URL("https://reputation.alienvault.com/reputation.generic"), "csv", false,0); + Boolean enabled = false; + List iocTypes = List.of("ipv4-addr"); + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); + String id = "alienvault_reputation_ip_database"; + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + id, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + urlDownloadSource, + null, + null, + schedule, + null, + null, + null, + null, + enabled, + iocTypes, false + ); + + // update default source config with enabled_for_scan updated + Response response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + id, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.OK, restStatus(response)); + Map updateResponseAsMap = asMap(response); + assertNotNull(updateResponseAsMap); + assertTrue(updateResponseAsMap.containsKey("source_config")); + HashMap scr = (HashMap) updateResponseAsMap.get("source_config"); + assertTrue(scr.containsKey("enabled")); + assertFalse((Boolean) scr.get("enabled")); + assertTrue(scr.containsKey("enabled_for_scan")); + assertFalse((Boolean) scr.get("enabled_for_scan")); + + // Ensure that only 1 ioc index is present from default source + List findingIndices = getIocIndices(); + Assert.assertEquals(1, findingIndices.size()); + + // try to update default source config again to ensure operation is not accepted when enabled_for_scan is unchanged + try { + makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + id, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("unsupported_operation_exception")); + } + // activate source + saTifSourceConfigDto = new SATIFSourceConfigDto( + id, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + urlDownloadSource, + null, + null, + schedule, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + + // update default source config with enabled_for_scan updated + response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + id, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.OK, restStatus(response)); + updateResponseAsMap = asMap(response); + assertNotNull(updateResponseAsMap); + assertTrue(updateResponseAsMap.containsKey("source_config")); + scr = (HashMap) updateResponseAsMap.get("source_config"); + assertTrue(scr.containsKey("enabled")); + assertTrue((Boolean) scr.get("enabled")); + assertTrue(scr.containsKey("enabled_for_scan")); + assertTrue((Boolean) scr.get("enabled_for_scan")); + } + + public void testDeleteIocUploadSourceConfigAndAllIocs() throws IOException { + String feedName = "test_ioc_upload"; + String feedFormat = "STIX"; + SourceConfigType sourceConfigType = SourceConfigType.IOC_UPLOAD; + + List iocs = List.of(new STIX2IOCDto( + "id", + "name", + new IOCType(IOCType.IPV4_TYPE), + "value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L)); + + IocUploadSource iocUploadSource = new IocUploadSource(null, iocs); + Boolean enabled = false; + List iocTypes = List.of(IOCType.IPV4_TYPE); + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + iocUploadSource, + null, + null, + null, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(response)); + Map responseBody = asMap(response); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, createdId), response.getHeader("Location")); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(1, hits.size()); + + // Delete source config + response = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/" + createdId, Collections.emptyMap(), null); + Assert.assertEquals(RestStatus.OK, restStatus(response)); + responseBody = asMap(response); + + String id = responseBody.get("_id").toString(); + assertEquals(id, createdId); + + // ensure all source configs are deleted + hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(0, hits.size()); + + // ensure all iocs are deleted + hits = executeSearch(IOC_ALL_INDEX_PATTERN, request); + Assert.assertEquals(0, hits.size()); + + // ensure that lock is deleted + hits = executeSearch(LOCK_INDEX_NAME,request); + Assert.assertEquals(0, hits.size()); + } + + public void testRefreshIocUploadSourceConfigFailure() throws IOException { + String feedName = "test_ioc_upload"; + String feedFormat = "STIX"; + SourceConfigType sourceConfigType = SourceConfigType.IOC_UPLOAD; + + List iocs = List.of(new STIX2IOCDto( + "id", + "name", + new IOCType(IOCType.IPV4_TYPE), + "value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L)); + + IocUploadSource iocUploadSource = new IocUploadSource(null, iocs); + Boolean enabled = false; + List iocTypes = List.of(IOCType.IPV4_TYPE); + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + iocUploadSource, + null, + null, + null, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(response)); + Map responseBody = asMap(response); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, createdId), response.getHeader("Location")); + + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(1, hits.size()); + + // Try to execute refresh api + try { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/" + createdId + "/_refresh", Collections.emptyMap(), null); + } catch (ResponseException ex) { + Assert.assertEquals(RestStatus.BAD_REQUEST, restStatus(ex.getResponse())); + } + } + + public void testSearchIocUploadSourceConfig() throws IOException { + String feedName = "test_ioc_upload"; + String feedFormat = "STIX"; + SourceConfigType sourceConfigType = SourceConfigType.IOC_UPLOAD; + + List iocs = List.of(new STIX2IOCDto( + "id", + "name", + new IOCType(IOCType.IPV4_TYPE), + "value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L)); + + IocUploadSource iocUploadSource = new IocUploadSource(null, iocs); + Boolean enabled = false; + List iocTypes = List.of(IOCType.IPV4_TYPE); + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + iocUploadSource, + null, + null, + null, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(response)); + Map responseBody = asMap(response); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, createdId), response.getHeader("Location")); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(1, hits.size()); + + // Search all source configs + Response sourceConfigResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/_search", Collections.emptyMap(), new StringEntity(request), new BasicHeader("Content-type", "application/json")); + Assert.assertEquals(RestStatus.OK, restStatus(sourceConfigResponse)); + Map respMap = asMap(sourceConfigResponse); + + // Expected value is 2 - one ioc upload source config and one default source config + Assert.assertEquals(2, ((Map) ((Map) respMap.get("hits")).get("total")).get("value")); + } + + public void testSearchAndCreateDefaultSourceConfig() throws IOException { + // Search source configs when none are created + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + + // Search all source configs + Response sourceConfigResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/_search", Collections.emptyMap(), new StringEntity(request), new BasicHeader("Content-type", "application/json")); + Assert.assertEquals(RestStatus.OK, restStatus(sourceConfigResponse)); + Map responseBody = asMap(sourceConfigResponse); + + // Expected value is 1 - only default source config + Assert.assertEquals(1, ((Map) ((Map) responseBody.get("hits")).get("total")).get("value")); + } + + public void testUpdateDefaultSourceConfigThrowsError() throws IOException { + // Search source configs when none are created + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + + // Search all source configs + Response sourceConfigResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/_search", Collections.emptyMap(), new StringEntity(request), new BasicHeader("Content-type", "application/json")); + Assert.assertEquals(RestStatus.OK, restStatus(sourceConfigResponse)); + Map responseBody = asMap(sourceConfigResponse); + + // Expected value is 1 - only default source config + Assert.assertEquals(1, ((Map) ((Map) responseBody.get("hits")).get("total")).get("value")); + + // Update default source config + String feedName = "test_update_default"; + String feedFormat = "STIX"; + SourceConfigType sourceConfigType = SourceConfigType.URL_DOWNLOAD; + UrlDownloadSource urlDownloadSource = new UrlDownloadSource(new URL("https://reputation.alienvault.com/reputation.generic"), "csv", false,0); + Boolean enabled = false; + List iocTypes = List.of("ipv4-addr"); + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); + String id = "alienvault_reputation_ip_database"; + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + id, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + urlDownloadSource, + null, + null, + schedule, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + + // update default source config + try { + makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + id, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("unsupported_operation_exception")); + } + + // update default source config again to ensure lock was released + try { + makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + id, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("unsupported_operation_exception")); + } + } + + public void testUpdateJobIndexMapping() throws IOException { + // Create job index with old threat intel mapping + // Try catch needed because of warning when creating a system index which is needed to replicate previous tif job mapping + try { + createIndex(JOB_INDEX_NAME, Settings.EMPTY, oldThreatIntelJobMapping()); + } catch (WarningFailureException e) { + // Ensure index was created with old mappings + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(0, hits.size()); + + Map props = getIndexMappingsAPIFlat(JOB_INDEX_NAME); + assertTrue(props.containsKey("enabled_time")); + assertTrue(props.containsKey("schedule.interval.start_time")); + assertFalse(props.containsKey("source_config.source.ioc_upload.file_name")); + assertFalse(props.containsKey("source_config.source.s3.object_key")); + } + + // Create new threat intel source config + SATIFSourceConfigDto saTifSourceConfigDto = getSatifSourceConfigDto(); + + Response makeResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(makeResponse)); + Map responseBody = asMap(makeResponse); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertTrue("incorrect version", createdVersion > 0); + + // Ensure source config document was indexed + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(1, hits.size()); + + // Ensure index mappings were updated + Map props = getIndexMappingsAPIFlat(JOB_INDEX_NAME); + assertTrue(props.containsKey("source_config.source.ioc_upload.file_name")); + assertTrue(props.containsKey("source_config.source.s3.object_key")); + assertTrue(props.containsKey("source_config.description")); + assertTrue(props.containsKey("source_config.last_update_time")); + assertTrue(props.containsKey("source_config.refresh_type")); + } + + private static SATIFSourceConfigDto getSatifSourceConfigDto() { + String feedName = "test_ioc_upload"; + String feedFormat = "STIX"; + SourceConfigType sourceConfigType = SourceConfigType.IOC_UPLOAD; + + List iocs = List.of(new STIX2IOCDto( + "id", + "name", + new IOCType(IOCType.IPV4_TYPE), + "value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L)); + + IocUploadSource iocUploadSource = new IocUploadSource(null, iocs); + Boolean enabled = false; + List iocTypes = List.of(IOCType.IPV4_TYPE); + return new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + iocUploadSource, + null, + null, + null, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + } + + @Override + protected boolean preserveIndicesUponCompletion() { + return false; + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/TestS3ConnectionRestIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/TestS3ConnectionRestIT.java new file mode 100644 index 000000000..9b58e0de2 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/TestS3ConnectionRestIT.java @@ -0,0 +1,220 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.resthandler; + +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.opensearch.client.Response; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.TestHelpers; +import org.opensearch.securityanalytics.action.TestS3ConnectionRequest; +import org.opensearch.securityanalytics.action.TestS3ConnectionResponse; +import org.opensearch.securityanalytics.commons.utils.testUtils.S3ObjectGenerator; +import org.opensearch.securityanalytics.util.STIX2IOCGenerator; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; + +import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.TEST_S3_CONNECTION_URI; + +/** + * The following system parameters must be specified to successfully run these tests: + * + * tests.TestS3ConnectionRestIT.bucketName - the name of the S3 bucket to use for the tests + * tests.TestS3ConnectionRestIT.objectKey - OPTIONAL - the key for the bucket object we want to check + * tests.TestS3ConnectionRestIT.region - the AWS region of the S3 bucket + * tests.TestS3ConnectionRestIT.roleArn - the IAM role ARN to assume when making S3 calls + * + * The local system must have sufficient credentials to write to S3, delete from S3, and assume the provided role. + * + * These tests are disabled by default as there is no default value for the tests.s3connector.bucket system property. This is + * intentional as the tests will fail when run without the proper setup, such as during CI workflows. + * + * Example command to manually run this class's ITs: + * ./gradlew ':integTest' --tests "org.opensearch.securityanalytics.resthandler.TestS3ConnectionRestIT" \ + * -Dtests.TestS3ConnectionRestIT.bucketName= \ + * -Dtests.TestS3ConnectionRestIT.objectKey= \ + * -Dtests.TestS3ConnectionRestIT.region= \ + * -Dtests.TestS3ConnectionRestIT.roleArn= + */ +@EnabledIfSystemProperty(named = "tests.TestS3ConnectionRestIT.bucketName", matches = ".+") +public class TestS3ConnectionRestIT extends SecurityAnalyticsRestTestCase { + private String bucketName; + private String objectKey; + private String region; + private String roleArn; + private S3Client s3Client; + private S3ObjectGenerator s3ObjectGenerator; + private STIX2IOCGenerator stix2IOCGenerator; + private TestS3ConnectionRequest request; + private boolean objectKeyProvided = false; + + @Before + public void initSource() throws IOException { + // Retrieve system parameters needed to run the tests + if (bucketName == null) { + bucketName = System.getProperty("tests.TestS3ConnectionRestIT.bucketName"); + objectKey = System.getProperty("tests.TestS3ConnectionRestIT.objectKey"); + region = System.getProperty("tests.TestS3ConnectionRestIT.region"); + roleArn = System.getProperty("tests.TestS3ConnectionRestIT.roleArn"); + objectKeyProvided = objectKey != null; + } + + // Only create the s3Client once + if (s3Client == null) { + s3Client = S3Client.builder() + .region(Region.of(region)) + .build(); + s3ObjectGenerator = new S3ObjectGenerator(s3Client, bucketName); + } + + // If objectKey isn't provided as system parameter, generate the objectKey in the bucket + if (!objectKeyProvided) { + objectKey = TestHelpers.randomLowerCaseString(); + stix2IOCGenerator = new STIX2IOCGenerator(); + s3ObjectGenerator.write(1, objectKey, stix2IOCGenerator); + } + } + + @After + public void afterTest() { + s3Client.close(); + } + + @Ignore + public void testConnection_succeeds() throws IOException { + // Create the test request + request = new TestS3ConnectionRequest(bucketName, objectKey, region, roleArn); + + // Execute test case + Response response = makeRequest(client(), "POST", TEST_S3_CONNECTION_URI, Collections.emptyMap(), toHttpEntity(request)); + + // Evaluate response + Map responseBody = asMap(response); + + String status = responseBody.get(TestS3ConnectionResponse.STATUS_FIELD).toString(); + assertEquals(RestStatus.OK.name(), status); + + String error = responseBody.get(TestS3ConnectionResponse.ERROR_FIELD).toString(); + assertTrue(error.isEmpty()); + } + + @Ignore + public void testConnection_wrongBucket() throws IOException { + // Create the test request + request = new TestS3ConnectionRequest("fakebucket", objectKey, region, roleArn); + + // Execute test case + Response response = makeRequest(client(), "POST", TEST_S3_CONNECTION_URI, Collections.emptyMap(), toHttpEntity(request)); + + // Evaluate response + Map responseBody = asMap(response); + + String status = responseBody.get(TestS3ConnectionResponse.STATUS_FIELD).toString(); + assertEquals(RestStatus.MOVED_PERMANENTLY.name(), status); + + String error = responseBody.get(TestS3ConnectionResponse.ERROR_FIELD).toString(); + assertEquals("Resource not found.", error); + } + + @Ignore + public void testConnection_wrongKey() throws IOException { + // Create the test request + request = new TestS3ConnectionRequest(bucketName, "fakekey", region, roleArn); + + // Execute test case + Response response = makeRequest(client(), "POST", TEST_S3_CONNECTION_URI, Collections.emptyMap(), toHttpEntity(request)); + + // Evaluate response + Map responseBody = asMap(response); + + String status = responseBody.get(TestS3ConnectionResponse.STATUS_FIELD).toString(); + assertEquals(RestStatus.NOT_FOUND.name(), status); + + String error = responseBody.get(TestS3ConnectionResponse.ERROR_FIELD).toString(); + assertEquals("The specified key does not exist.", error); + } + + @Ignore + public void testConnection_wrongRegion() throws IOException { + // Create the test request + String wrongRegion = (Objects.equals(region, "us-west-2")) ? "us-east-1" : "us-west-2"; + request = new TestS3ConnectionRequest(bucketName, objectKey, wrongRegion, roleArn); + + // Execute test case + Response response = makeRequest(client(), "POST", TEST_S3_CONNECTION_URI, Collections.emptyMap(), toHttpEntity(request)); + + // Evaluate response + Map responseBody = asMap(response); + + String status = responseBody.get(TestS3ConnectionResponse.STATUS_FIELD).toString(); + assertEquals(RestStatus.BAD_REQUEST.name(), status); + + String error = responseBody.get(TestS3ConnectionResponse.ERROR_FIELD).toString(); + assertEquals("Resource not found.", error); + } + + @Ignore + public void testConnection_invalidRegion() throws IOException { + // Create the test request + request = new TestS3ConnectionRequest(bucketName, objectKey, "fa-ke-1", roleArn); + + // Execute test case + Response response = makeRequest(client(), "POST", TEST_S3_CONNECTION_URI, Collections.emptyMap(), toHttpEntity(request)); + + // Evaluate response + Map responseBody = asMap(response); + + String status = responseBody.get(TestS3ConnectionResponse.STATUS_FIELD).toString(); + assertEquals(RestStatus.BAD_REQUEST.name(), status); + + String error = responseBody.get(TestS3ConnectionResponse.ERROR_FIELD).toString(); + assertEquals("Resource not found.", error); + } + + @Ignore + public void testConnection_wrongRoleArn() throws IOException { + // Create the test request + request = new TestS3ConnectionRequest(bucketName, objectKey, region, "arn:aws:iam::123456789012:role/iam-fake-role"); + + // Execute test case + Response response = makeRequest(client(), "POST", TEST_S3_CONNECTION_URI, Collections.emptyMap(), toHttpEntity(request)); + + // Evaluate response + Map responseBody = asMap(response); + + String status = responseBody.get(TestS3ConnectionResponse.STATUS_FIELD).toString(); + assertEquals(RestStatus.FORBIDDEN.name(), status); + + String error = responseBody.get(TestS3ConnectionResponse.ERROR_FIELD).toString(); + assertTrue(error.contains("is not authorized to perform: sts:AssumeRole on resource")); + } + + @Ignore + public void testConnection_invalidRoleArn() throws IOException { + // Create the test request + request = new TestS3ConnectionRequest(bucketName, objectKey, region, "arn:aws:iam::12345:role/iam-invalid-role"); + + // Execute test case + Response response = makeRequest(client(), "POST", TEST_S3_CONNECTION_URI, Collections.emptyMap(), toHttpEntity(request)); + + // Evaluate response + Map responseBody = asMap(response); + + String status = responseBody.get(TestS3ConnectionResponse.STATUS_FIELD).toString(); + assertEquals(RestStatus.FORBIDDEN.name(), status); + + String error = responseBody.get(TestS3ConnectionResponse.ERROR_FIELD).toString(); + assertTrue(error.contains("is not authorized to perform: sts:AssumeRole on resource")); + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelAlertIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelAlertIT.java new file mode 100644 index 000000000..f1b7ac7e3 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelAlertIT.java @@ -0,0 +1,106 @@ +package org.opensearch.securityanalytics.resthandler; + +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.junit.Assert; +import org.opensearch.client.Response; +import org.opensearch.commons.alerting.model.Alert; +import org.opensearch.commons.alerting.model.Monitor; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.model.threatintel.ThreatIntelAlert; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.opensearch.securityanalytics.TestHelpers.randomIndex; +import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; +import static org.opensearch.securityanalytics.resthandler.ThreatIntelMonitorRestApiIT.randomIocScanMonitorDto; +import static org.opensearch.securityanalytics.threatIntel.iocscan.dao.ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME; + +public class ThreatIntelAlertIT extends SecurityAnalyticsRestTestCase { + public void testStatusUpdateFromAcknowledgedToComplete() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDto(index); + Response response = makeRequest(client(), + "POST", + SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, + Collections.emptyMap(), + toHttpEntity(iocScanMonitor)); + Map responseBody = asMap(response); + final String monitorId = responseBody.get("id").toString(); + Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, monitorId); + List alertIds = indexThreatIntelAlerts(monitorId, Alert.State.ACKNOWLEDGED); + Response updateStatusResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_ALERTS_STATUS_URI, + Map.of("alert_ids", String.join(",", alertIds), "state", "COMPLETED"), null); + Map updateStatusResponseMap = responseAsMap(updateStatusResponse); + ArrayList> updatedAlerts = (ArrayList>) updateStatusResponseMap.get("updated_alerts"); + assertEquals(2, updatedAlerts.size()); + assertTrue(alertIds.contains(updatedAlerts.get(0).get("id").toString())); + assertTrue(alertIds.contains(updatedAlerts.get(1).get("id").toString())); + assertEquals(Alert.State.COMPLETED.toString(), updatedAlerts.get(0).get("state").toString()); + assertEquals(Alert.State.COMPLETED.toString(), updatedAlerts.get(1).get("state").toString()); + } + + public void testStatusUpdateFromActiveToAcknowledged() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDto(index); + Response response = makeRequest(client(), + "POST", + SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, + Collections.emptyMap(), + toHttpEntity(iocScanMonitor)); + Map responseBody = asMap(response); + final String monitorId = responseBody.get("id").toString(); + Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, monitorId); + List alertIds = indexThreatIntelAlerts(monitorId, Alert.State.ACTIVE); + Response updateStatusResponseEntity = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_ALERTS_STATUS_URI, + Map.of("alert_ids", String.join(",", alertIds), "state", "ACKNOWLEDGED"), null); + Map updateResponseMap = responseAsMap(updateStatusResponseEntity); + ArrayList> updatedAlerts = (ArrayList>) updateResponseMap.get("updated_alerts"); + assertEquals(2, updatedAlerts.size()); + assertTrue(alertIds.contains(updatedAlerts.get(0).get("id").toString())); + assertTrue(alertIds.contains(updatedAlerts.get(1).get("id").toString())); + assertEquals(Alert.State.ACKNOWLEDGED.toString(), updatedAlerts.get(0).get("state").toString()); + assertEquals(Alert.State.ACKNOWLEDGED.toString(), updatedAlerts.get(1).get("state").toString()); + } + + private List indexThreatIntelAlerts(String monitorId, Alert.State state) throws IOException { + List ids = new ArrayList<>(); + int i = 2; + while (i-- > 0) { + ThreatIntelAlert alert = new ThreatIntelAlert( + randomAlphaOfLength(10), + 1, + 1, + null, + randomAlphaOfLength(10), + randomAlphaOfLength(10), + monitorId, + randomAlphaOfLength(10), + state, + Instant.now(), + null, + Instant.now(), + Instant.now(), + null, + "high", + randomAlphaOfLength(10), + "ip", + Collections.emptyList(), + List.of(randomAlphaOfLength(10)) + ); + ids.add(alert.getId()); + makeRequest(client(), "POST", THREAT_INTEL_ALERT_ALIAS_NAME + "/_doc/" + alert.getId() + "?refresh", Map.of(), + new StringEntity(toJsonString(alert), ContentType.APPLICATION_JSON)); + + } + return ids; + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java new file mode 100644 index 000000000..77fafd157 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java @@ -0,0 +1,1042 @@ +package org.opensearch.securityanalytics.resthandler; + +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.HttpStatus; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.Assert; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.commons.alerting.model.IntervalSchedule; +import org.opensearch.commons.alerting.model.Monitor; +import org.opensearch.commons.alerting.model.action.Action; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.search.SearchHit; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsActionRequest; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.securityanalytics.model.DetectorTrigger; +import org.opensearch.securityanalytics.model.STIX2IOCDto; +import org.opensearch.securityanalytics.threatIntel.common.RefreshType; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.ThreatIntelAlertService; +import org.opensearch.securityanalytics.threatIntel.iocscan.dto.PerIocTypeScanInputDto; +import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelTriggerDto; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.util.Collections.emptyList; +import static org.opensearch.securityanalytics.TestHelpers.randomAction; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithTriggers; +import static org.opensearch.securityanalytics.TestHelpers.randomIndex; +import static org.opensearch.securityanalytics.TestHelpers.randomThreatInteMonitorAction; +import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; +import static org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestSearchThreatIntelMonitorAction.SEARCH_THREAT_INTEL_MONITOR_PATH; + +public class ThreatIntelMonitorRestApiIT extends SecurityAnalyticsRestTestCase { + private final Logger log = LogManager.getLogger(ThreatIntelMonitorRestApiIT.class); + + private List testIocDtos = new ArrayList<>(); + + public String indexSourceConfigsAndIocs(List iocVals) throws IOException { + testIocDtos = new ArrayList<>(); + for (int i1 = 0; i1 < iocVals.size(); i1++) { + // create IOCs + STIX2IOCDto stix2IOCDto = new STIX2IOCDto( + "id" + i1, + "random", + new IOCType(IOCType.IPV4_TYPE), + iocVals.get(i1), + "", + Instant.now(), + Instant.now(), + "", + emptyList(), + "spec", + "configId", + "", + 1L + ); + + testIocDtos.add(stix2IOCDto); + } + return indexTifSourceConfig(testIocDtos); + } + + public String indexSourceConfigsAndIocs(List ipVals, List hashVals, List domainVals) throws IOException { + testIocDtos = new ArrayList<>(); + for (int i1 = 0; i1 < ipVals.size(); i1++) { + // create IOCs + STIX2IOCDto stix2IOCDto = new STIX2IOCDto( + "id" + randomAlphaOfLength(3), + "random", + new IOCType(IOCType.IPV4_TYPE), + ipVals.get(i1), + "", + Instant.now(), + Instant.now(), + "", + emptyList(), + "spec", + "configId", + "", + 1L + ); + + testIocDtos.add(stix2IOCDto); + } + for (int i1 = 0; i1 < hashVals.size(); i1++) { + // create IOCs + STIX2IOCDto stix2IOCDto = new STIX2IOCDto( + "id" + randomAlphaOfLength(3), + "random", + new IOCType(IOCType.HASHES_TYPE), + hashVals.get(i1), + "", + Instant.now(), + Instant.now(), + "", + emptyList(), + "spec", + "configId", + "", + 1L + ); + + testIocDtos.add(stix2IOCDto); + } + for (int i1 = 0; i1 < domainVals.size(); i1++) { + // create IOCs + STIX2IOCDto stix2IOCDto = new STIX2IOCDto( + "id" + randomAlphaOfLength(3), + "random", + new IOCType(IOCType.DOMAIN_NAME_TYPE), + domainVals.get(i1), + "", + Instant.now(), + Instant.now(), + "", + emptyList(), + "spec", + "configId", + "", + 1L + ); + + testIocDtos.add(stix2IOCDto); + } + return indexTifSourceConfig(testIocDtos); + } + + private String indexTifSourceConfig(List testIocDtos) throws IOException { + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + "configId", + SATIFSourceConfig.NO_VERSION, + "name1", + "STIX2", + SourceConfigType.IOC_UPLOAD, + "description", + null, + Instant.now(), + new IocUploadSource(null, testIocDtos), + null, + Instant.now(), + null, + TIFJobState.AVAILABLE, + RefreshType.FULL, + null, + null, + false, + List.of(IOCType.IPV4_TYPE, IOCType.HASHES_TYPE, IOCType.DOMAIN_NAME_TYPE), + true + ); + + Response makeResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(makeResponse)); + + Assert.assertEquals(201, makeResponse.getStatusLine().getStatusCode()); + Map responseBody = asMap(makeResponse); + return responseBody.get("_id").toString(); + } + + public void testCreateThreatIntelMonitor_monitorAliases() throws IOException { + updateClusterSetting(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT.getKey(), "1"); + Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + Map responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(0, ((List>) responseAsMap.get("ioc_findings")).size()); + List vals = List.of("ip1", "ip2"); + String createdId = indexSourceConfigsAndIocs(vals); + + String index = "alias1"; + Map> testAlias = createTestAlias(index, 1, true); + String monitorName = "test_monitor_name"; + + /**create monitor */ + ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDto(index); + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + try { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + fail(); + } catch (Exception e) { + /** creating a second threat intel monitor should fail*/ + assertTrue(e.getMessage().contains("already exists")); + } + + final String monitorId = responseBody.get("id").toString(); + Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, monitorId); + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Assert.assertEquals(200, executeResponse.getStatusLine().getStatusCode()); + + Response alertingMonitorResponse = getAlertingMonitor(client(), monitorId); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + int i = 1; + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + + executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + String matchAllRequest = getMatchAllRequest(); + Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + HashMap hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + HashMap totalHits = (HashMap) hits.get("total"); + Integer totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 1); + makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); + + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(2, ((List>) responseAsMap.get("ioc_findings")).size()); + + //alerts + List searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(4, searchHits.size()); + + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + executeAlertingMonitor(monitorId, Collections.emptyMap()); + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(4, ((List>) responseAsMap.get("ioc_findings")).size()); + + // Use ListIOCs API to confirm expected number of findings are returned + String listIocsUri = String.format("?%s=%s", ListIOCsActionRequest.FEED_IDS_FIELD, createdId); + Response listIocsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.LIST_IOCS_URI + listIocsUri, Collections.emptyMap(), null); + Map listIocsResponseMap = responseAsMap(listIocsResponse); + List> iocsMap = (List>) listIocsResponseMap.get("iocs"); + assertEquals(2, iocsMap.size()); + iocsMap.forEach((iocDetails) -> { + String iocId = (String) iocDetails.get("id"); + int numFindings = (Integer) iocDetails.get("num_findings"); + assertTrue(testIocDtos.stream().anyMatch(ioc -> iocId.equals(ioc.getId()))); + assertEquals(2, numFindings); + }); + + // Use ListIOCs API with large size to ensure matchQuery related bug is not throwing too many bool clauses exception + listIocsUri = String.format("?%s=%s", "size", 1000); + listIocsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.LIST_IOCS_URI, Collections.emptyMap(), null); + assertEquals(200, listIocsResponse.getStatusLine().getStatusCode()); + listIocsResponseMap = responseAsMap(listIocsResponse); + iocsMap = (List>) listIocsResponseMap.get("iocs"); + assertTrue(2 < iocsMap.size()); // number should be greater than custom source iocs because of default config + + //alerts via system index search + searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(4, searchHits.size()); + + // alerts via API + Map params = new HashMap<>(); + Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_ALERTS_URI, params, null); + Map getAlertsBody = asMap(getAlertsResponse); + Assert.assertEquals(4, getAlertsBody.get("total_alerts")); + + + ThreatIntelMonitorDto updateMonitorDto = new ThreatIntelMonitorDto( + monitorId, + iocScanMonitor.getName() + "update", + iocScanMonitor.getPerIocTypeScanInputList(), + new IntervalSchedule(5, ChronoUnit.MINUTES, Instant.now()), + false, + null, + List.of(iocScanMonitor.getTriggers().get(0), iocScanMonitor.getTriggers().get(1)) + ); + //update monitor + response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), toHttpEntity(updateMonitorDto)); + Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + responseBody = asMap(response); + assertEquals(responseBody.get("id").toString(), monitorId); + assertEquals(((HashMap) responseBody.get("monitor")).get("name").toString(), iocScanMonitor.getName() + "update"); + + //delete + Response delete = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), null); + Assert.assertEquals(200, delete.getStatusLine().getStatusCode()); + + searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + totalHits = (HashMap) hits.get("total"); + totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 0); + } + + public void testCreateThreatIntelMonitor_configureMultipleIndicatorTypesInMonitor() throws IOException { + updateClusterSetting(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT.getKey(), "1"); + Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + Map responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(0, ((List>) responseAsMap.get("ioc_findings")).size()); + List ipVals = List.of("ip1", "ip2"); + List hashVals = List.of("h1", "h2"); + List domainVals = List.of("d1", "d2"); + String createdId = indexSourceConfigsAndIocs(ipVals, hashVals, domainVals); + + String ipIndex = "ipAlias"; + createTestAlias(ipIndex, 1, true); + String hashIndex = "hashAlias"; + createTestAlias(hashIndex, 1, true); + String domainIndex = "domainAlias"; + createTestAlias(domainIndex, 1, true); + + + /**create monitor */ + ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDtoWithMultipleIndicatorTypesToScan(ipIndex, hashIndex, domainIndex); + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + try { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + fail(); + } catch (Exception e) { + /** creating a second threat intel monitor should fail*/ + assertTrue(e.getMessage().contains("already exists")); + } + + final String monitorId = responseBody.get("id").toString(); + Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, monitorId); + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Assert.assertEquals(200, executeResponse.getStatusLine().getStatusCode()); + + Response alertingMonitorResponse = getAlertingMonitor(client(), monitorId); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + int i = 1; + for (String val : ipVals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(ipIndex, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + for (String val : hashVals) { + String doc = String.format("{\"hash\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(hashIndex, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + for (String val : domainVals) { + String doc = String.format("{\"domain\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(domainIndex, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + + executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + String matchAllRequest = getMatchAllRequest(); + Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + HashMap hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + HashMap totalHits = (HashMap) hits.get("total"); + Integer totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 1); + makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); + + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(6, ((List>) responseAsMap.get("ioc_findings")).size()); + + //alerts + List searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(6, searchHits.size()); + } + + public void testCreateThreatIntelMonitor() throws IOException { + updateClusterSetting(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT.getKey(), "1"); + Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + Map responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(0, ((List>) responseAsMap.get("ioc_findings")).size()); + List vals = List.of("ip1", "ip2"); + String createdId = indexSourceConfigsAndIocs(vals); + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + String monitorName = "test_monitor_name"; + + + /**create monitor */ + ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDto(index); + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + try { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + fail(); + } catch (Exception e) { + /** creating a second threat intel monitor should fail*/ + assertTrue(e.getMessage().contains("already exists")); + } + + final String monitorId = responseBody.get("id").toString(); + Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, monitorId); + + Response alertingMonitorResponse = getAlertingMonitor(client(), monitorId); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + int i = 1; + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + indexDoc(index, "" + i++, String.format("{\"ip\":\"1.2.3.4\", \"ip1\":\"1.2.3.4\"}", val, val)); + indexDoc(index, "" + i++, String.format("{\"random\":\"%s\", \"random1\":\"%s\"}", val, val)); + } catch (IOException e) { + fail(); + } + } + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + String matchAllRequest = getMatchAllRequest(); + Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + HashMap hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + HashMap totalHits = (HashMap) hits.get("total"); + Integer totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 1); + makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); + + + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(2, ((List>) responseAsMap.get("ioc_findings")).size()); + + //alerts + List searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(4, searchHits.size()); + + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + executeAlertingMonitor(monitorId, Collections.emptyMap()); + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(4, ((List>) responseAsMap.get("ioc_findings")).size()); + + // Use ListIOCs API to confirm expected number of findings are returned + String listIocsUri = String.format("?%s=%s", ListIOCsActionRequest.FEED_IDS_FIELD, createdId); + Response listIocsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.LIST_IOCS_URI + listIocsUri, Collections.emptyMap(), null); + Map listIocsResponseMap = responseAsMap(listIocsResponse); + List> iocsMap = (List>) listIocsResponseMap.get("iocs"); + assertEquals(2, iocsMap.size()); + iocsMap.forEach((iocDetails) -> { + String iocId = (String) iocDetails.get("id"); + int numFindings = (Integer) iocDetails.get("num_findings"); + assertTrue(testIocDtos.stream().anyMatch(ioc -> iocId.equals(ioc.getId()))); + assertEquals(2, numFindings); + }); + + //alerts via system index search + searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(4, searchHits.size()); + + // alerts via API + Map params = new HashMap<>(); + Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_ALERTS_URI, params, null); + Map getAlertsBody = asMap(getAlertsResponse); + Assert.assertEquals(4, getAlertsBody.get("total_alerts")); + + + ThreatIntelMonitorDto updateMonitorDto = new ThreatIntelMonitorDto( + monitorId, + iocScanMonitor.getName() + "update", + iocScanMonitor.getPerIocTypeScanInputList(), + new IntervalSchedule(5, ChronoUnit.MINUTES, Instant.now()), + false, + null, + List.of(iocScanMonitor.getTriggers().get(0), iocScanMonitor.getTriggers().get(1)) + ); + //update monitor + response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), toHttpEntity(updateMonitorDto)); + Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + responseBody = asMap(response); + assertEquals(responseBody.get("id").toString(), monitorId); + assertEquals(((HashMap) responseBody.get("monitor")).get("name").toString(), iocScanMonitor.getName() + "update"); + + //delete + Response delete = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), null); + Assert.assertEquals(200, delete.getStatusLine().getStatusCode()); + + searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + totalHits = (HashMap) hits.get("total"); + totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 0); + } + + // verify scenario where findings are generated but alert creation fails - monitor execution should be marked successful + public void testCreateThreatIntelMonitor_testExecution_findingSucceedsButAlertFails_ExecutionSucceeds() throws IOException { + updateClusterSetting(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT.getKey(), "1"); + Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + Map responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(0, ((List>) responseAsMap.get("ioc_findings")).size()); + List vals = List.of("ip1", "ip2"); + String createdId = indexSourceConfigsAndIocs(vals); + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + String monitorName = "test_monitor_name"; + + + /**create monitor */ + ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDto(index); + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + try { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + fail(); + } catch (Exception e) { + /** creating a second threat intel monitor should fail*/ + assertTrue(e.getMessage().contains("already exists")); + } + + final String monitorId = responseBody.get("id").toString(); + Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, monitorId); + + Response alertingMonitorResponse = getAlertingMonitor(client(), monitorId); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + int i = 1; + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + indexDoc(index, "" + i++, String.format("{\"ip\":\"1.2.3.4\", \"ip1\":\"1.2.3.4\"}", val, val)); + indexDoc(index, "" + i++, String.format("{\"random\":\"%s\", \"random1\":\"%s\"}", val, val)); + } catch (IOException e) { + fail(); + } + } + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + String matchAllRequest = getMatchAllRequest(); + Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, + ContentType.APPLICATION_JSON.getMimeType(), ContentType.APPLICATION_JSON.getCharset().name())); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + HashMap hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + HashMap totalHits = (HashMap) hits.get("total"); + Integer totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 1); + makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, + ContentType.APPLICATION_JSON.getMimeType(), ContentType.APPLICATION_JSON.getCharset().name())); + + + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(2, ((List>) responseAsMap.get("ioc_findings")).size()); + + //alerts + List searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(4, searchHits.size()); + + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + closeIndex(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME); + executeAlertingMonitor(monitorId, Collections.emptyMap()); + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(4, ((List>) responseAsMap.get("ioc_findings")).size()); + openIndex(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME); + // Use ListIOCs API to confirm expected number of findings are returned + String listIocsUri = String.format("?%s=%s", ListIOCsActionRequest.FEED_IDS_FIELD, createdId); + Response listIocsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.LIST_IOCS_URI + listIocsUri, Collections.emptyMap(), null); + Map listIocsResponseMap = responseAsMap(listIocsResponse); + List> iocsMap = (List>) listIocsResponseMap.get("iocs"); + assertEquals(2, iocsMap.size()); + iocsMap.forEach((iocDetails) -> { + String iocId = (String) iocDetails.get("id"); + int numFindings = (Integer) iocDetails.get("num_findings"); + assertTrue(testIocDtos.stream().anyMatch(ioc -> iocId.equals(ioc.getId()))); + assertEquals(2, numFindings); + }); + + //alerts via system index search + searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(4, searchHits.size()); + + // alerts via API + Map params = new HashMap<>(); + Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_ALERTS_URI, params, null); + Map getAlertsBody = asMap(getAlertsResponse); + Assert.assertEquals(4, getAlertsBody.get("total_alerts")); + + + ThreatIntelMonitorDto updateMonitorDto = new ThreatIntelMonitorDto( + monitorId, + iocScanMonitor.getName() + "update", + iocScanMonitor.getPerIocTypeScanInputList(), + new IntervalSchedule(5, ChronoUnit.MINUTES, Instant.now()), + false, + null, + List.of(iocScanMonitor.getTriggers().get(0), iocScanMonitor.getTriggers().get(1)) + ); + //update monitor + response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), toHttpEntity(updateMonitorDto)); + Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + responseBody = asMap(response); + assertEquals(responseBody.get("id").toString(), monitorId); + assertEquals(((HashMap) responseBody.get("monitor")).get("name").toString(), iocScanMonitor.getName() + "update"); + + //delete + Response delete = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), null); + Assert.assertEquals(200, delete.getStatusLine().getStatusCode()); + + searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, + ContentType.APPLICATION_JSON.getMimeType(), ContentType.APPLICATION_JSON.getCharset().name())); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + totalHits = (HashMap) hits.get("total"); + totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 0); + } + + // verify scenario where alerts and findings are generated but notification fails - monitor execution should be marked successful + public void testCreateThreatIntelMonitor_testNotifications_invaliConfigExecutionSucceeds() throws IOException { + + updateClusterSetting(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT.getKey(), "1"); + Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + Map responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(0, ((List>) responseAsMap.get("ioc_findings")).size()); + List vals = List.of("ip1", "ip2"); + String createdId = indexSourceConfigsAndIocs(vals); + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + String monitorName = "test_monitor_name"; + + Action triggerAction1 = randomThreatInteMonitorAction(randomAlphaOfLength(10)); + Action triggerAction2 = randomThreatInteMonitorAction(randomAlphaOfLength(10)); + /**create monitor with trigger*/ + ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDtoWithTriggers(index, List.of(triggerAction1, triggerAction2)); + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + try { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + fail(); + } catch (Exception e) { + /** creating a second threat intel monitor should fail*/ + assertTrue(e.getMessage().contains("already exists")); + } + + final String monitorId = responseBody.get("id").toString(); + Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, monitorId); + + Response alertingMonitorResponse = getAlertingMonitor(client(), monitorId); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + int i = 1; + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + indexDoc(index, "" + i++, String.format("{\"ip\":\"1.2.3.4\", \"ip1\":\"1.2.3.4\"}", val, val)); + indexDoc(index, "" + i++, String.format("{\"random\":\"%s\", \"random1\":\"%s\"}", val, val)); + } catch (IOException e) { + fail(); + } + } + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + String matchAllRequest = getMatchAllRequest(); + Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, + ContentType.APPLICATION_JSON.getMimeType(), ContentType.APPLICATION_JSON.getCharset().name())); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + HashMap hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + HashMap totalHits = (HashMap) hits.get("total"); + Integer totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 1); + makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, + ContentType.APPLICATION_JSON.getMimeType(), ContentType.APPLICATION_JSON.getCharset().name())); + + + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(2, ((List>) responseAsMap.get("ioc_findings")).size()); + + //alerts + List searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(2, searchHits.size()); + + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + executeAlertingMonitor(monitorId, Collections.emptyMap()); + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(4, ((List>) responseAsMap.get("ioc_findings")).size()); + + // Use ListIOCs API to confirm expected number of findings are returned + String listIocsUri = String.format("?%s=%s", ListIOCsActionRequest.FEED_IDS_FIELD, createdId); + Response listIocsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.LIST_IOCS_URI + listIocsUri, Collections.emptyMap(), null); + Map listIocsResponseMap = responseAsMap(listIocsResponse); + List> iocsMap = (List>) listIocsResponseMap.get("iocs"); + assertEquals(2, iocsMap.size()); + iocsMap.forEach((iocDetails) -> { + String iocId = (String) iocDetails.get("id"); + int numFindings = (Integer) iocDetails.get("num_findings"); + assertTrue(testIocDtos.stream().anyMatch(ioc -> iocId.equals(ioc.getId()))); + assertEquals(2, numFindings); + }); + + //alerts via system index search + searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(2, searchHits.size()); + + // alerts via API + Map params = new HashMap<>(); + Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_ALERTS_URI, params, null); + Map getAlertsBody = asMap(getAlertsResponse); + Assert.assertEquals(2, getAlertsBody.get("total_alerts")); + + + ThreatIntelMonitorDto updateMonitorDto = new ThreatIntelMonitorDto( + monitorId, + iocScanMonitor.getName() + "update", + iocScanMonitor.getPerIocTypeScanInputList(), + new IntervalSchedule(5, ChronoUnit.MINUTES, Instant.now()), + false, + null, + List.of(iocScanMonitor.getTriggers().get(0)) + ); + //update monitor + response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), toHttpEntity(updateMonitorDto)); + Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + responseBody = asMap(response); + assertEquals(responseBody.get("id").toString(), monitorId); + assertEquals(((HashMap) responseBody.get("monitor")).get("name").toString(), iocScanMonitor.getName() + "update"); + + //delete + Response delete = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), null); + Assert.assertEquals(200, delete.getStatusLine().getStatusCode()); + + searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, + ContentType.APPLICATION_JSON.getMimeType(), ContentType.APPLICATION_JSON.getCharset().name())); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + totalHits = (HashMap) hits.get("total"); + totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 0); + } + + public void testCreateThreatIntelMonitorWithExistingDetector() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + Map responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(0, ((List>) responseAsMap.get("ioc_findings")).size()); + List vals = List.of("ip1", "ip2"); + String createdId = indexSourceConfigsAndIocs(vals); + String monitorName = "test_monitor_name"; + + + /**create monitor */ + ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDto(index); + response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + try { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + fail(); + } catch (Exception e) { + /** creating a second threat intel monitor should fail*/ + assertTrue(e.getMessage().contains("already exists")); + } + + final String monitorId = responseBody.get("id").toString(); + Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, monitorId); + + Response alertingMonitorResponse = getAlertingMonitor(client(), monitorId); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + int i = 1; + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + String matchAllRequest = getMatchAllRequest(); + Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + HashMap hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + HashMap totalHits = (HashMap) hits.get("total"); + Integer totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 1); + makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); + + + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(2, ((List>) responseAsMap.get("ioc_findings")).size()); + + //alerts + List searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(4, searchHits.size()); + + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + executeAlertingMonitor(monitorId, Collections.emptyMap()); + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(4, ((List>) responseAsMap.get("ioc_findings")).size()); + + // Use ListIOCs API to confirm expected number of findings are returned + String listIocsUri = String.format("?%s=%s", ListIOCsActionRequest.FEED_IDS_FIELD, createdId); + Response listIocsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.LIST_IOCS_URI + listIocsUri, Collections.emptyMap(), null); + Map listIocsResponseMap = responseAsMap(listIocsResponse); + List> iocsMap = (List>) listIocsResponseMap.get("iocs"); + assertEquals(2, iocsMap.size()); + iocsMap.forEach((iocDetails) -> { + String iocId = (String) iocDetails.get("id"); + int numFindings = (Integer) iocDetails.get("num_findings"); + assertTrue(testIocDtos.stream().anyMatch(ioc -> iocId.equals(ioc.getId()))); + assertEquals(2, numFindings); + }); + + //alerts via system index search + searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(4, searchHits.size()); + + // alerts via API + Map params = new HashMap<>(); + Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_ALERTS_URI, params, null); + Map getAlertsBody = asMap(getAlertsResponse); + Assert.assertEquals(4, getAlertsBody.get("total_alerts")); + + + ThreatIntelMonitorDto updateMonitorDto = new ThreatIntelMonitorDto( + monitorId, + iocScanMonitor.getName() + "update", + iocScanMonitor.getPerIocTypeScanInputList(), + new IntervalSchedule(5, ChronoUnit.MINUTES, Instant.now()), + false, + null, + List.of(iocScanMonitor.getTriggers().get(0), iocScanMonitor.getTriggers().get(1)) + ); + //update monitor + response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), toHttpEntity(updateMonitorDto)); + Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + responseBody = asMap(response); + assertEquals(responseBody.get("id").toString(), monitorId); + assertEquals(((HashMap) responseBody.get("monitor")).get("name").toString(), iocScanMonitor.getName() + "update"); + + //delete + Response delete = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), null); + Assert.assertEquals(200, delete.getStatusLine().getStatusCode()); + + searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + totalHits = (HashMap) hits.get("total"); + totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 0); + } + + public void testCreateThreatIntelMonitor_invalidMonitorJson() throws IOException { + ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDto("test-index"); + + String monitorJson = iocScanMonitor.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).toString(); + final String invalidMonitorJson = monitorJson.replace("\"interval\":1", "\"interval\":100000000000000000000000000000000000000"); + + ResponseException exception = Assert.assertThrows(ResponseException.class, + () -> makeRequest( + client(), + "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, + Collections.emptyMap(), + new StringEntity(invalidMonitorJson, ContentType.APPLICATION_JSON) + ) + ); + Assert.assertTrue(exception.getMessage().contains("Failed to parse threat intel monitor: ")); + Assert.assertTrue(exception.getMessage().contains("\"status\":400")); + } + + public static String getMatchAllRequest() { + return "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + } + + public static ThreatIntelMonitorDto randomIocScanMonitorDto(String index) { + ThreatIntelTriggerDto t1 = new ThreatIntelTriggerDto(List.of(index, "randomIndex"), List.of(IOCType.IPV4_TYPE, IOCType.DOMAIN_NAME_TYPE), emptyList(), "match", null, "severity"); + ThreatIntelTriggerDto t2 = new ThreatIntelTriggerDto(List.of("randomIndex"), List.of(IOCType.DOMAIN_NAME_TYPE), emptyList(), "nomatch", null, "severity"); + ThreatIntelTriggerDto t3 = new ThreatIntelTriggerDto(emptyList(), List.of(IOCType.DOMAIN_NAME_TYPE), emptyList(), "domainmatchsonomatch", null, "severity"); + ThreatIntelTriggerDto t4 = new ThreatIntelTriggerDto(List.of(index), emptyList(), emptyList(), "indexmatch", null, "severity"); + + return new ThreatIntelMonitorDto( + Monitor.NO_ID, + randomAlphaOfLength(10), + List.of(new PerIocTypeScanInputDto(IOCType.IPV4_TYPE, Map.of(index, List.of("ip")))), + new IntervalSchedule(1, ChronoUnit.MINUTES, Instant.now()), + false, + null, + List.of(t1, t2, t3, t4)); + } + + public static ThreatIntelMonitorDto randomIocScanMonitorDtoWithTriggers(String index, List actions) { + ThreatIntelTriggerDto t1 = new ThreatIntelTriggerDto(List.of(), List.of(IOCType.IPV4_TYPE, IOCType.DOMAIN_NAME_TYPE), actions, "match", null, "severity"); + + return new ThreatIntelMonitorDto( + Monitor.NO_ID, + randomAlphaOfLength(10), + List.of(new PerIocTypeScanInputDto(IOCType.IPV4_TYPE, Map.of(index, List.of("ip")))), + new IntervalSchedule(1, ChronoUnit.MINUTES, Instant.now()), + false, + null, + List.of(t1)); + } + + public static ThreatIntelMonitorDto randomIocScanMonitorDtoWithMultipleIndicatorTypesToScan(String ipIndex, String hashIndex, String domainIndex) { + ThreatIntelTriggerDto t1 = new ThreatIntelTriggerDto(List.of(ipIndex, "randomIndex"), List.of(IOCType.IPV4_TYPE, IOCType.DOMAIN_NAME_TYPE), emptyList(), "match", null, "severity"); + ThreatIntelTriggerDto t2 = new ThreatIntelTriggerDto(List.of("randomIndex"), List.of(IOCType.DOMAIN_NAME_TYPE), emptyList(), "nomatch", null, "severity"); + ThreatIntelTriggerDto t3 = new ThreatIntelTriggerDto(emptyList(), List.of(IOCType.DOMAIN_NAME_TYPE), emptyList(), "domainmatchsonomatch", null, "severity"); + ThreatIntelTriggerDto t4 = new ThreatIntelTriggerDto(List.of(ipIndex), emptyList(), emptyList(), "indexmatch", null, "severity"); + + return new ThreatIntelMonitorDto( + Monitor.NO_ID, + randomAlphaOfLength(10), + List.of( + new PerIocTypeScanInputDto(IOCType.IPV4_TYPE, Map.of(ipIndex, List.of("ip"))), + new PerIocTypeScanInputDto(IOCType.HASHES_TYPE, Map.of(hashIndex, List.of("hash"))), + new PerIocTypeScanInputDto(IOCType.DOMAIN_NAME_TYPE, Map.of(domainIndex, List.of("domain"))) + ), + new IntervalSchedule(1, ChronoUnit.MINUTES, Instant.now()), + false, + null, + List.of(t1, t2, t3, t4)); + } + + @Override + protected boolean preserveIndicesUponCompletion() { + return false; + } +} + diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java index 20d36ab2d..d62ea5888 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java @@ -31,9 +31,11 @@ import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobUpdateService; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService; +import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; +import org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskListener; import org.opensearch.test.client.NoOpNodeClient; diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobActionTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobActionTests.java index 27a01f5c0..f8c6ecadc 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobActionTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobActionTests.java @@ -5,25 +5,6 @@ package org.opensearch.securityanalytics.threatIntel.action; -import org.junit.Before; -import org.mockito.ArgumentCaptor; -import org.opensearch.action.StepListener; -import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.core.action.ActionListener; -import org.opensearch.jobscheduler.spi.LockModel; -import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; -import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; -import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; -import org.opensearch.tasks.Task; -import org.opensearch.securityanalytics.TestHelpers; - -import java.io.IOException; -import java.util.ConcurrentModificationException; - -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.*; - /*public class TransportPutTIFJobActionTests extends ThreatIntelTestCase { private TransportPutTIFJobAction action; diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/common/ThreatIntelLockServiceTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/common/ThreatIntelLockServiceTests.java index 7a95e746f..2f5aa90a3 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/common/ThreatIntelLockServiceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/common/ThreatIntelLockServiceTests.java @@ -12,6 +12,7 @@ import java.time.Instant; import java.util.concurrent.atomic.AtomicReference; +import org.junit.Assert; import org.junit.Before; import org.opensearch.action.DocWriteResponse; import org.opensearch.action.update.UpdateRequest; @@ -52,7 +53,16 @@ public void testAcquireLock_whenCalled_thenNotBlocked() { public void testReleaseLock_whenValidInput_thenSucceed() { // Cannot test because LockService is final class // Simply calling method to increase coverage - noOpsLockService.releaseLock(null); + LockModel lockModel = new LockModel( + TestHelpers.randomLowerCaseString(), + TestHelpers.randomLowerCaseString(), + Instant.now(), + LOCK_DURATION_IN_SECONDS, + false + ); + noOpsLockService.releaseLock(lockModel, ActionListener.wrap( + Assert::assertFalse, e -> fail() + )); } public void testRenewLock_whenCalled_thenNotBlocked() { diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java index d9a9eea94..720738d7b 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java @@ -28,7 +28,7 @@ import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; import java.io.IOException; import java.time.Instant; @@ -45,7 +45,7 @@ import static org.opensearch.securityanalytics.TestHelpers.*; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.TIF_UPDATE_INTERVAL; -import static org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataUtils.getTifdList; +import static org.opensearch.securityanalytics.threatIntel.util.ThreatIntelFeedDataUtils.getTifdList; public class ThreatIntelJobRunnerIT extends SecurityAnalyticsRestTestCase { private static final Logger log = LogManager.getLogger(ThreatIntelJobRunnerIT.class); diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceRestApiIT.java new file mode 100644 index 000000000..79548fd48 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceRestApiIT.java @@ -0,0 +1,135 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.iocscan.dao; + +import org.junit.Assert; +import org.opensearch.client.Response; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; +import org.opensearch.securityanalytics.model.threatintel.IocWithFeeds; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class IocFindingServiceRestApiIT extends SecurityAnalyticsRestTestCase { + + @SuppressWarnings("unchecked") + public void testGetIocFindings() throws IOException { + makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search?startIndex=1&size=5", + Map.of(), null); + List iocFindings = generateIocMatches(10); + for (IocFinding iocFinding: iocFindings) { + makeRequest(client(), "POST", IocFindingService.IOC_FINDING_ALIAS_NAME + "/_doc?refresh", Map.of(), + toHttpEntity(iocFinding)); + } + + Response response = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search?startIndex=1&size=5", + Map.of(), null); + Map responseAsMap = responseAsMap(response); + Assert.assertEquals(5, ((List>) responseAsMap.get("ioc_findings")).size()); + } + + @SuppressWarnings("unchecked") + public void testGetIocFindingsWithIocIdFilter() throws IOException { + makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search?startIndex=1&size=5", + Map.of(), null); + List iocFindings = generateIocMatches(10); + for (IocFinding iocFinding: iocFindings) { + makeRequest(client(), "POST", IocFindingService.IOC_FINDING_ALIAS_NAME + "/_doc?refresh", Map.of(), + toHttpEntity(iocFinding)); + } + String iocId = iocFindings.stream().map(iocFinding -> iocFinding.getFeedIds().get(0).getIocId()).findFirst().get(); + + Response response = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search?iocIds=" + iocId, + Map.of(), null); + Map responseAsMap = responseAsMap(response); + Assert.assertEquals(1, ((List>) responseAsMap.get("ioc_findings")).size()); + } +// +// public void testGetIocFindingsRolloverByMaxDocs() throws IOException, InterruptedException { +// updateClusterSetting(IOC_FINDING_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); +// updateClusterSetting(IOC_FINDING_HISTORY_MAX_DOCS.getKey(), "1"); +// makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search?startIndex=1&size=5", +// Map.of(), null); +// List iocFindings = generateIocMatches(5); +// for (IocFinding iocFinding: iocFindings) { +// makeRequest(client(), "POST", IocFindingService.IOC_FINDING_ALIAS_NAME + "/_doc?refresh", Map.of(), +// toHttpEntity(iocFinding)); +// } +// +// AtomicBoolean found = new AtomicBoolean(false); +// OpenSearchTestCase.waitUntil(() -> { +// try { +// found.set(getIocFindingIndices().size() == 2); +// return found.get(); +// } catch (IOException e) { +// return false; +// } +// }, 30000, TimeUnit.SECONDS); +// Assert.assertTrue(found.get()); +// } +// +// public void testGetIocFindingsRolloverByMaxAge() throws IOException, InterruptedException { +// updateClusterSetting(IOC_FINDING_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); +// updateClusterSetting(IOC_FINDING_HISTORY_MAX_DOCS.getKey(), "1000"); +// updateClusterSetting(IOC_FINDING_HISTORY_INDEX_MAX_AGE.getKey(), "1s"); +// makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search?startIndex=1&size=5", +// Map.of(), null); +// List iocFindings = generateIocMatches(5); +// for (IocFinding iocFinding: iocFindings) { +// makeRequest(client(), "POST", IocFindingService.IOC_FINDING_ALIAS_NAME + "/_doc?refresh", Map.of(), +// toHttpEntity(iocFinding)); +// } +// +// AtomicBoolean found = new AtomicBoolean(false); +// OpenSearchTestCase.waitUntil(() -> { +// try { +// found.set(getIocFindingIndices().size() == 2); +// return found.get(); +// } catch (IOException e) { +// return false; +// } +// }, 30000, TimeUnit.SECONDS); +// Assert.assertTrue(found.get()); +// +// updateClusterSetting(IOC_FINDING_HISTORY_INDEX_MAX_AGE.getKey(), "1000s"); +// updateClusterSetting(IOC_FINDING_HISTORY_RETENTION_PERIOD.getKey(), "1s"); +// +// AtomicBoolean retFound = new AtomicBoolean(false); +// OpenSearchTestCase.waitUntil(() -> { +// try { +// retFound.set(getIocFindingIndices().size() == 1); +// return retFound.get(); +// } catch (IOException e) { +// return false; +// } +// }, 30000, TimeUnit.SECONDS); +// Assert.assertTrue(retFound.get()); +// } + + private List generateIocMatches(int i) { + List iocFindings = new ArrayList<>(); + String monitorId = randomAlphaOfLength(10); + String monitorName = randomAlphaOfLength(10); + for (int i1 = 0; i1 < i; i1++) { + iocFindings.add(new IocFinding( + randomAlphaOfLength(10), + randomList(1, 10, () -> randomAlphaOfLength(10)),//docids + randomList(1, 10, () -> new IocWithFeeds(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10))), //feedids + monitorId, + monitorName, + randomAlphaOfLength(10), + "IP", + Instant.now(), + randomAlphaOfLength(10) + )); + } + return iocFindings; + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java index f7b7ff8d1..1d7f1706c 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java @@ -10,17 +10,16 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; import org.opensearch.securityanalytics.TestHelpers; -import org.opensearch.securityanalytics.model.DetectorTrigger; import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.List; import java.util.Locale; -import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; +import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; public class TIFJobParameterTests extends ThreatIntelTestCase { private static final Logger log = LogManager.getLogger(TIFJobParameterTests.class); diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java index 71bd68c61..ec13b7635 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java @@ -6,23 +6,7 @@ package org.opensearch.securityanalytics.threatIntel.jobscheduler; -import org.junit.Before; -import org.opensearch.jobscheduler.spi.JobDocVersion; -import org.opensearch.jobscheduler.spi.JobExecutionContext; -import org.opensearch.jobscheduler.spi.LockModel; -import org.opensearch.jobscheduler.spi.ScheduledJobParameter; -import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService; -import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; -import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; -import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; -import org.opensearch.securityanalytics.TestHelpers; - -import java.io.IOException; -import java.time.Instant; -import java.util.Optional; - import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; /*public class TIFJobRunnerTests extends ThreatIntelTestCase { @Before diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/model/ThreatIntelSourceTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/model/ThreatIntelSourceTests.java new file mode 100644 index 000000000..dd6c13b07 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/model/ThreatIntelSourceTests.java @@ -0,0 +1,73 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.model; + +import org.junit.Test; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.TestHelpers; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +public class ThreatIntelSourceTests extends OpenSearchTestCase { + + @Test + public void testParseWithS3Source() throws IOException { + String sourceString = "{\n" + + " \"s3\": {\n" + + " \"bucket_name\": \"bucket-name\",\n" + + " \"object_key\": \"object-key\",\n" + + " \"region\": \"us-west-2\",\n" + + " \"role_arn\": \"arn:aws:iam::123456789012:role/test_role\"\n" + + " }\n" + + " }"; + Source source = Source.parse(TestHelpers.parser(sourceString)); + assertSame(source.getClass(), S3Source.class); + assertEquals("bucket-name", ((S3Source) source).getBucketName()); + assertEquals("object-key", ((S3Source) source).getObjectKey()); + assertEquals("us-west-2", ((S3Source) source).getRegion()); + assertEquals("arn:aws:iam::123456789012:role/test_role", ((S3Source) source).getRoleArn()); + } + + @Test + public void testParseWithIocUploadSource() throws IOException { + String sourceString = "{\n" + + " \"ioc_upload\" : {\n" + + " \"iocs\": []\n" + + " }\n" + + " }"; + Source source = Source.parse(TestHelpers.parser(sourceString)); + assertSame(source.getClass(), IocUploadSource.class); + assertTrue(((IocUploadSource) source).getIocs().isEmpty()); + } + + @Test + public void testParseWithUrlDownloadSource() throws IOException { + String sourceString = "{\n" + + " \"url_download\": {\n" + + " \"url\": \"https://reputation.alienvault.com/reputation.generic\",\n" + + " \"feed_format\": \"csv\"\n" + + " }\n" + + " }"; + Source source = Source.parse(TestHelpers.parser(sourceString)); + assertSame(source.getClass(), UrlDownloadSource.class); + assertEquals("https://reputation.alienvault.com/reputation.generic", ((UrlDownloadSource) source).getUrl().toString()); + assertEquals("csv", ((UrlDownloadSource) source).getFeedFormat()); + } + + @Test + public void testParseInvalidSourceField() { + String sourceString = "{\n" + + " \"invalid_field\" : {\n" + + " \"iocs\": []\n" + + " }"; + + SecurityAnalyticsException exception = assertThrows(SecurityAnalyticsException.class, () -> Source.parse(TestHelpers.parser(sourceString))); + assertEquals(RestStatus.BAD_REQUEST, exception.status()); + assertTrue(exception.getMessage().contains("Unexpected input in 'source' field when reading ioc store config.")); + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java new file mode 100644 index 000000000..d56969de0 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java @@ -0,0 +1,97 @@ +package org.opensearch.securityanalytics.threatIntel.model.monitor; + + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.commons.alerting.model.DataSources; +import org.opensearch.commons.alerting.model.DocLevelMonitorInput; +import org.opensearch.commons.alerting.model.IntervalSchedule; +import org.opensearch.commons.alerting.model.Monitor; +import org.opensearch.commons.alerting.model.remote.monitors.RemoteDocLevelMonitorInput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Map; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE; + +public class ThreatIntelInputTests extends OpenSearchTestCase { + + public void testThreatInputSerde() throws IOException { + ThreatIntelInput threatIntelInput = getThreatIntelInput(); + BytesStreamOutput out = new BytesStreamOutput(); + threatIntelInput.writeTo(out); + BytesReference bytes = out.bytes(); + Monitor monitor = new Monitor( + randomAlphaOfLength(10), + Monitor.NO_VERSION, + randomAlphaOfLength(10), + true, + new IntervalSchedule(1, ChronoUnit.MINUTES, null), + Instant.now(), + Instant.now(), + THREAT_INTEL_MONITOR_TYPE, + null, + 4, + List.of( + new RemoteDocLevelMonitorInput( + bytes, + new DocLevelMonitorInput("threat intel input", + List.of("index1", "index2"), + emptyList(), + true + ) + ) + ), + emptyList(), + emptyMap(), + new DataSources(), + false, + null, + "security_analytics" + ); + BytesStreamOutput monitorOut = new BytesStreamOutput(); + monitor.writeTo(monitorOut); + + StreamInput sin = StreamInput.wrap(monitorOut.bytes().toBytesRef().bytes); + String monitorString = toJsonString(monitor); + Monitor parsedMonitor = Monitor.parse(getParser(monitorString)); + assertEquals(((RemoteDocLevelMonitorInput) parsedMonitor.getInputs().get(0)).getInput(), ((RemoteDocLevelMonitorInput) parsedMonitor.getInputs().get(0)).getInput()); + } + + private ThreatIntelInput getThreatIntelInput() { + return new ThreatIntelInput(randomList(randomInt(5), () -> randomPerIocTypeThreatIntel())); + } + + private PerIocTypeScanInput randomPerIocTypeThreatIntel() { + return new PerIocTypeScanInput( + randomAlphaOfLength(10), + Map.of("index1", List.of("f1", "f2"), "index2", List.of("f3", "f4")) + ); + } + + public XContentParser getParser(String xc) throws IOException { + XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); + parser.nextToken(); + return parser; + + } + + public String toJsonString(Monitor monitor) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + return monitor.toXContent(builder, ToXContent.EMPTY_PARAMS).toString(); + + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/util/STIX2IOCGenerator.java b/src/test/java/org/opensearch/securityanalytics/util/STIX2IOCGenerator.java new file mode 100644 index 000000000..6c040d9a2 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/util/STIX2IOCGenerator.java @@ -0,0 +1,294 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.util; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.commons.model.IOC; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.commons.utils.testUtils.PojoGenerator; +import org.opensearch.securityanalytics.model.DetailedSTIX2IOCDto; +import org.opensearch.securityanalytics.model.STIX2IOC; +import org.opensearch.securityanalytics.model.STIX2IOCDto; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.opensearch.securityanalytics.TestHelpers.randomLowerCaseString; +import static org.opensearch.test.OpenSearchTestCase.randomInt; +import static org.opensearch.test.OpenSearchTestCase.randomLong; + +public class STIX2IOCGenerator implements PojoGenerator { + private List iocs; + private List types = IOCType.types.stream().map(IOCType::new).collect(Collectors.toList()); + + private final ObjectMapper objectMapper; + + public STIX2IOCGenerator() { + this.objectMapper = new ObjectMapper(); + } + + public STIX2IOCGenerator(List types) { + this(); + this.types = types; + } + + @Override + public void write(final int numberOfIOCs, final OutputStream outputStream) { + try (final PrintWriter printWriter = new PrintWriter(outputStream)) { + writeLines(numberOfIOCs, printWriter); + } + } + + /** + * For each IOCType in 'types', 'numberOfIOCs' will be generated in the bucket object. + * Defaults to generating 'numberOfIOCs' of each IOCType. + * @param numberOfIOCs the number of each IOCType to generate in the bucket object. + * @param printWriter prints formatted representations of objects to a text-output stream. + */ + private void writeLines(final int numberOfIOCs, final PrintWriter printWriter) { + final List iocs = new ArrayList<>(); + for (IOCType type : types) { + final List newIocs = IntStream.range(0, numberOfIOCs) + .mapToObj(i -> randomIOC(type)) + .collect(Collectors.toList()); + iocs.addAll(newIocs); + } + this.iocs = iocs; + iocs.forEach(ioc -> writeLine(ioc, printWriter)); + } + + private void writeLine(final IOC ioc, final PrintWriter printWriter) { + try { + final String iocAsString; + if (ioc.getClass() == STIX2IOC.class) { + iocAsString = BytesReference.bytes(((STIX2IOC) ioc).toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)).utf8ToString(); + } else { + iocAsString = objectMapper.writeValueAsString(ioc); + } + printWriter.write(iocAsString + "\n"); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static STIX2IOC randomIOC(IOCType type) { + return randomIOC( + null, + null, + type, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); + } + + public static STIX2IOC randomIOC() { + return randomIOC(null); + } + + public List getIocs() { + return iocs; + } + + public List getTypes() { + return types; + } + + public static STIX2IOC randomIOC( + String id, + String name, + IOCType type, + String value, + String severity, + Instant created, + Instant modified, + String description, + List labels, + String specVersion, + String feedId, + String feedName, + Long version + ) { + if (name == null) { + name = randomLowerCaseString(); + } + if (type == null) { + type = new IOCType(IOCType.types.get(randomInt(IOCType.types.size() - 1))); + } + if (value == null) { + value = randomLowerCaseString(); + } + if (severity == null) { + severity = randomLowerCaseString(); + } + if (created == null) { + created = Instant.now(); + } + if (modified == null) { + modified = Instant.now().plusSeconds(3600); // 1 hour + } + if (description == null) { + description = randomLowerCaseString(); + } + if (labels == null) { + labels = IntStream.range(0, randomInt(5)) + .mapToObj(i -> randomLowerCaseString()) + .collect(Collectors.toList()); + } + if (specVersion == null) { + specVersion = randomLowerCaseString(); + } + if (feedId == null) { + feedId = randomLowerCaseString(); + } + if (feedName == null) { + feedName = randomLowerCaseString(); + } + if (version == null) { + version = randomLong(); + } + + return new STIX2IOC( + id, + name, + type, + value, + severity, + created, + modified, + description, + labels, + specVersion, + feedId, + feedName, + version + ); + } + + public static STIX2IOCDto randomIocDto() { + return new STIX2IOCDto(randomIOC()); + } + + public static STIX2IOCDto randomIocDto(IOCType type) { + return new STIX2IOCDto(randomIOC(type)); + } + + public static STIX2IOCDto randomIocDto( + String id, + String name, + IOCType type, + String value, + String severity, + Instant created, + Instant modified, + String description, + List labels, + String specVersion, + String feedId, + String feedName, + Long version + ) { + return new STIX2IOCDto(randomIOC( + id, + name, + type, + value, + severity, + created, + modified, + description, + labels, + specVersion, + feedId, + feedName, + version + )); + } + + public static String toJsonString(STIX2IOC ioc) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder = ioc.toXContent(builder, ToXContent.EMPTY_PARAMS); + return BytesReference.bytes(builder).utf8ToString(); + } + + public static String toJsonString(STIX2IOCDto ioc) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder = ioc.toXContent(builder, ToXContent.EMPTY_PARAMS); + return BytesReference.bytes(builder).utf8ToString(); + } + + public static String toJsonString(DetailedSTIX2IOCDto ioc) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder = ioc.toXContent(builder, ToXContent.EMPTY_PARAMS); + return BytesReference.bytes(builder).utf8ToString(); + } + + public static void assertIOCEqualsDTO(STIX2IOC ioc, STIX2IOCDto iocDto) { + STIX2IOC newIoc = new STIX2IOC(iocDto); + assertEqualIOCs(ioc, newIoc); + } + + public static void assertEqualIOCs(STIX2IOC ioc, STIX2IOC newIoc) { + assertNotNull(newIoc.getId()); + assertEquals(ioc.getName(), newIoc.getName()); + assertEquals(ioc.getType().toString(), newIoc.getType().toString()); + assertEquals(ioc.getValue(), newIoc.getValue()); + assertEquals(ioc.getSeverity(), newIoc.getSeverity()); +// assertEquals(ioc.getCreated(), newIoc.getCreated()); +// assertEquals(ioc.getModified(), newIoc.getModified()); + assertEquals(ioc.getDescription(), newIoc.getDescription()); + assertEquals(ioc.getLabels(), newIoc.getLabels()); + assertEquals(ioc.getSpecVersion(), newIoc.getSpecVersion()); + assertEquals(ioc.getFeedId(), newIoc.getFeedId()); + assertEquals(ioc.getFeedName(), newIoc.getFeedName()); + } + + public static void assertEqualIocDtos(STIX2IOCDto ioc, STIX2IOCDto newIoc) { + assertNotNull(newIoc.getId()); + assertEquals(ioc.getName(), newIoc.getName()); + assertEquals(ioc.getType().toString(), newIoc.getType().toString()); + assertEquals(ioc.getValue(), newIoc.getValue()); + assertEquals(ioc.getSeverity(), newIoc.getSeverity()); +// assertEquals(ioc.getCreated(), newIoc.getCreated()); +// assertEquals(ioc.getModified(), newIoc.getModified()); + assertEquals(ioc.getDescription(), newIoc.getDescription()); + assertEquals(ioc.getLabels(), newIoc.getLabels()); + assertEquals(ioc.getSpecVersion(), newIoc.getSpecVersion()); + assertEquals(ioc.getFeedId(), newIoc.getFeedId()); + assertEquals(ioc.getFeedName(), newIoc.getFeedName()); + } + + public static void assertEqualIocDtos(DetailedSTIX2IOCDto ioc, DetailedSTIX2IOCDto newIoc) { + assertEqualIocDtos(ioc.getIoc(), newIoc.getIoc()); + assertEquals(ioc.getNumFindings(), newIoc.getNumFindings()); + } + + public static String getListIOCsURI() { + return String.format("%s", SecurityAnalyticsPlugin.LIST_IOCS_URI); + + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/writable/LogTypeTests.java b/src/test/java/org/opensearch/securityanalytics/writable/LogTypeTests.java index d9d592641..626ec6ac3 100644 --- a/src/test/java/org/opensearch/securityanalytics/writable/LogTypeTests.java +++ b/src/test/java/org/opensearch/securityanalytics/writable/LogTypeTests.java @@ -21,7 +21,7 @@ public class LogTypeTests { public void testLogTypeAsStreamRawFieldOnly() throws IOException { LogType logType = new LogType( "1", "my_log_type", "description", false, - List.of(new LogType.Mapping("rawField", null, null)), + List.of(new LogType.Mapping("rawField", null, null, null)), List.of(new LogType.IocFields("ip", List.of("dst.ip"))) ); BytesStreamOutput out = new BytesStreamOutput(); @@ -41,7 +41,7 @@ public void testLogTypeAsStreamRawFieldOnly() throws IOException { public void testLogTypeAsStreamFull() throws IOException { LogType logType = new LogType( "1", "my_log_type", "description", false, - List.of(new LogType.Mapping("rawField", "some_ecs_field", "some_ocsf_field")), + List.of(new LogType.Mapping("rawField", "some_ecs_field", "some_ocsf_field", "some_ocsf11_field")), List.of(new LogType.IocFields("ip", List.of("dst.ip"))) ); BytesStreamOutput out = new BytesStreamOutput(); diff --git a/src/test/resources/ad_ldap-sample.json b/src/test/resources/ad_ldap-sample.json index 3b89ca581..f6beef93c 100644 --- a/src/test/resources/ad_ldap-sample.json +++ b/src/test/resources/ad_ldap-sample.json @@ -2,13 +2,13 @@ "azure.signinlogs.properties.user_id": "1234", "azure.activitylogs.category": "1234", "azure.platformlogs.operation_name": "1234", - "modified_properties.new_value": "1234", + "ModifiedProperties.NewValue": "1234", "azure.resource.provider": "1234", "azure.signinlogs.properties.conditional_access_status": "1234", "SearchFilter": "1234", "azure.platformlogs.result_type": "1234", "azure.signinlogs.result_description": "1234", - "azure.signinlogs.properties.device_detail.is_compliant": "1234", + "azure.signinlogs.properties.device_detail.is_compliant": false, "resource_display_name": "1234", "azure.signinlogs.properties.authentication_requirement": "1234", "target_resources": "1234", @@ -18,5 +18,6 @@ "EventID": 12345, "azure.signinlogs.properties.network_location_details": "1234", "azure.auditlogs.properties.activity_display_name": "1234", - "creationTime": "2022-12-27T20:29:31.734Z" + "creationTime": "2022-12-27T20:29:31.734Z", + "DeviceDetail.isCompliant": true } diff --git a/src/test/resources/azure-sample.json b/src/test/resources/azure-sample.json index bccf2cf40..7c0facebd 100644 --- a/src/test/resources/azure-sample.json +++ b/src/test/resources/azure-sample.json @@ -1,27 +1,44 @@ { - "azure.signinlogs.props.user_id": "111", + "azure.signinlogs.properties.user_id": "111", "azure.activitylogs.category": "111", "modified_properties.new_value": "111", + "azure.activitylogs.identity.claims_initiated_by_user.name": "111", "azure.resource.provider": "111", - "azure.signinlogs.props.conditional_access_status": "111", + "azure.signinlogs.properties.conditional_access_status": "111", "SearchFilter": "111", "azure.platformlogs.result_type": "111", - "azure.signinlogs.props.device_detail.is_compliant": true, + "azure.signinlogs.properties.device_detail.is_compliant": true, "ResourceDisplayName": 111, - "azure.signinlogs.props.authentication_requirement": "111", + "azure.signinlogs.properties.resource_tenant_id": "111", + "azure.signinlogs.properties.authentication_requirement": "111", "TargetResources": "111", "Workload": "111", - "azure.signinlogs.props.device_detail.device_id": "111", + "azure.signinlogs.properties.device_detail.device_id": "111", "azure.platformlogs.operation_name": "111", - "azure.signinlogs.props.resource_id": "111", + "azure.signinlogs.properties.resource_id": "111", "EventID": 1234, - "azure.signinlogs.props.network_location_details": "111", - "azure.auditlogs.props.activity_display_name": "111", - "azure.signinlogs.result-description": "111", + "failure_status_reason": "111", + "azure.signinlogs.properties.network_location_details": "111", + "azure.auditlogs.properties.activity_display_name": "111", + "azure.signinlogs.result_description": "111", "eventSource": "111", "eventName": "111", "azure.platformlogs.status": "111", "azure.auditlogs.props.logged_by_service": "111", - "properties_message": "111", - "creationTime": "2022-12-27T20:29:31.734Z" + "properties.message": "111", + "creationTime": "2022-12-27T20:29:31.734Z", + "Count": "111", + "azure.signinlogs.properties.app_id": "111", + "azure.signinlogs.properties.client_app_used": true, + "ActivityDetails": "111", + "Target": "111", + "azure.signinlogs.properties.device_detail.trust_type": "111", + "azure.signinlogs.properties.home_tenant_id": "111", + "ConsentContext.IsAdminConsent": true, + "InitiatedBy": "111", + "azure.activitylogs.operation_name": "111", + "user_agent.name": "111", + "azure.signinlogs.properties.risk_state": "111", + "riskEventType": "111", + "azure.auditlogs.properties.logged_by_service": "111" } diff --git a/src/test/resources/cloudtrail-sample.json b/src/test/resources/cloudtrail-sample.json index 02e9ee2fb..e3c812f35 100644 --- a/src/test/resources/cloudtrail-sample.json +++ b/src/test/resources/cloudtrail-sample.json @@ -6,26 +6,31 @@ "source.as.organization.name": "213123", "source.ip": "213123", "userIdentity.arn": "213123", - "eventName": "213123", + "eventName": "DeleteIdentity", "eventType": "213123", "errorCode": "213123", - "eventSource": "213123", + "eventSource": "ses.amazonaws.com", "tlsDetails.tlsVersion": "213123", "user_agent.name": "213123", "threat.matched.providers": "213123", "aws-cloudtrail-event_name": "123", "aws-cloudtrail-event_source": "123", "aws-cloudtrail-event_type": "123", - "aws-cloudtrail-error_message": "123", + "aws.cloudtrail.error_message": "123", "aws-cloudtrail-error_code": "123", - "aws-cloudtrail-response_elements-text": "123", - "aws-cloudtrail-response_elements-pending_modified_values-master_user_password": "123", + "aws.cloudtrail.response_elements.text": "123", + "aws.cloudtrail.response_elements.pending_modified_values.master_user_password": "123", "aws-cloudtrail-response_elements-publicly_accessible": "123", "aws-cloudtrail-request_parameters-arn": "123", - "aws-cloudtrail-request_parameters-attribute": "123", + "aws.cloudtrail.request_parameters.attribute": "123", "aws-cloudtrail-request_parameters-username": "123", - "aws-cloudtrail-request_parameters-container_definitions-command": "123", - "aws-cloudtrail-user_identity-session_context-session_issuer-type": "123", + "aws.cloudtrail.request_parameters.container_definitions.command": "123", + "aws.cloudtrail.user_identity.session_context.session_issuer.type": "123", "aws-cloudtrail-user_identity-arn": "123", + "aws.cloudtrail.user_identity.type": "123", + "aws.cloudtrail.response_elements.publicly_accessible": "123", + "aws.cloudtrail.user_agent": "123", + "requestParameters": "123", + "type": "123", "eventTime": "2022-12-27T20:29:31.734Z" } diff --git a/src/test/resources/s3-sample.json b/src/test/resources/s3-sample.json index 440dcf184..68f84b042 100644 --- a/src/test/resources/s3-sample.json +++ b/src/test/resources/s3-sample.json @@ -8,7 +8,7 @@ "Operation": "123", "RequestURI_key": "123", "aws.s3access.requester": "1234", - "aws-cloudtrail-event_source": "123", - "aws-cloudtrail-event_name": "123", + "aws.cloudtrail.event_source": "123", + "aws.cloudtrail.event_name": "123", "eventTime": "123" } diff --git a/src/test/resources/waf-sample.json b/src/test/resources/waf-sample.json index 51a09cb4b..0cb24f5f0 100644 --- a/src/test/resources/waf-sample.json +++ b/src/test/resources/waf-sample.json @@ -53,5 +53,6 @@ { "name": "awswaf:managed:aws:bot-control:signal:known_bot_data_center" } - ] + ], + "waf.request.headers.user_agent": "WPScan v" }