Merge branch 'opensearch-project:main' into main

Author: brianf-aws

.github/workflows/maven-publish.yml

@@ -5,8 +5,8 @@ on:
   push:
     branches:
       - main
-      - 1.*
-      - 2.*
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.x'
 
 jobs:
   build-and-publish-snapshots:
@@ -33,8 +33,14 @@ jobs:
           export-env: true
         env:
           OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
-          SONATYPE_USERNAME: op://opensearch-infra-secrets/maven-central-portal-credentials/username
-          SONATYPE_PASSWORD: op://opensearch-infra-secrets/maven-central-portal-credentials/password
+          MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo
+          MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
+          aws-region: us-east-1
 
       - name: publish snapshots to maven
         run: |

.github/workflows/require-approval.yml

@@ -35,7 +35,15 @@ jobs:
             else
               # Check if the author is in the approvers list
               approvers=$(cat .github/CODEOWNERS | grep @ | tr -d '* ' | sed 's/@/,/g' | sed 's/,//1')
-              if [[ "$approvers" =~ "$author" ]]; then
+              is_approved=false
+              IFS=',' read -ra APPROVER_ARRAY <<< "$approvers"
+              for approver in "${APPROVER_ARRAY[@]}"; do
+                if [[ "$approver" == "$author" ]]; then
+                  is_approved=true
+                  break
+                fi
+              done
+              if [[ "$is_approved" == "true" ]]; then
                 echo "$author is in the approval list."
                 echo "is-require-approval=ml-commons-cicd-env" >> $GITHUB_OUTPUT
               else

.gitignore

@@ -10,3 +10,4 @@ ml-algorithms/build/
 plugin/build/
 .DS_Store
 */bin/
+**/*.factorypath

build-tools/repositories.gradle

@@ -5,8 +5,7 @@
 
 repositories {
     mavenLocal()
-    maven { url "https://central.sonatype.com/repository/maven-snapshots/" }
-    maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
+    maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" }
     mavenCentral()
     maven {url 'https://oss.sonatype.org/content/repositories/snapshots/'}
     maven { url "https://ci.opensearch.org/ci/dbc/snapshots/lucene/" }

build.gradle

@@ -11,7 +11,7 @@ buildscript {
     ext {
         opensearch_group = "org.opensearch"
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
-        opensearch_version = System.getProperty("opensearch.version", "3.3.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "3.4.0-SNAPSHOT")
         buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         asm_version = "9.7"
 
@@ -31,8 +31,7 @@ buildscript {
 
     repositories {
         mavenLocal()
-        maven { url "https://central.sonatype.com/repository/maven-snapshots/" }
-        maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
+        maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" }
         mavenCentral()
         maven { url "https://plugins.gradle.org/m2/" }
         maven { url "https://ci.opensearch.org/ci/dbc/snapshots/lucene/" }
@@ -97,6 +96,24 @@ subprojects {
         // Force spotless depending on newer version of guava due to CVE-2023-2976. Remove after spotless upgrades.
         resolutionStrategy.force "com.google.guava:guava:32.1.3-jre"
         resolutionStrategy.force 'org.apache.commons:commons-compress:1.26.0'
+        resolutionStrategy.force "io.netty:netty-buffer:${versions.netty}"
+        resolutionStrategy.force "io.netty:netty-codec:${versions.netty}"
+        resolutionStrategy.force "io.netty:netty-codec-http:${versions.netty}"
+        resolutionStrategy.force "io.netty:netty-codec-http2:${versions.netty}"
+        resolutionStrategy.force "io.netty:netty-common:${versions.netty}"
+        resolutionStrategy.force "io.netty:netty-handler:${versions.netty}"
+        resolutionStrategy.force "io.netty:netty-resolver:${versions.netty}"
+        resolutionStrategy.force "io.netty:netty-transport:${versions.netty}"
+        resolutionStrategy.force "io.netty:netty-transport-native-unix-common:${versions.netty}"
+        resolutionStrategy.force "software.amazon.awssdk:bom:${versions.aws}"
+        resolutionStrategy.force "software.amazon.awssdk:kms:${versions.aws}"
+        resolutionStrategy.force "software.amazon.awssdk:dynamodb:${versions.aws}"
+        resolutionStrategy.force "software.amazon.awssdk:dynamodb-enhanced:${versions.aws}"
+        resolutionStrategy.force "org.dafny:DafnyRuntime:4.9.0"
+    }
+
+    configurations.all {
+        exclude group: 'org.bouncycastle', module: 'bcprov-jdk18on'
     }
 
     apply plugin: 'com.diffplug.spotless'

client/build.gradle

@@ -72,10 +72,11 @@ publishing {
         }
         maven {
             name = "Snapshots"
-            url = "https://central.sonatype.com/repository/maven-snapshots/"
-            credentials {
-                username "$System.env.SONATYPE_USERNAME"
-                password "$System.env.SONATYPE_PASSWORD"
+            url = System.getenv("MAVEN_SNAPSHOTS_S3_REPO")
+            credentials(AwsCredentials) {
+                accessKey = System.getenv("AWS_ACCESS_KEY_ID")
+                secretKey = System.getenv("AWS_SECRET_ACCESS_KEY")
+                sessionToken = System.getenv("AWS_SESSION_TOKEN")
             }
         }
     }

common/build.gradle

@@ -22,8 +22,10 @@ dependencies {
     testImplementation group: 'org.mockito', name: 'mockito-core', version: '5.15.2'
     testImplementation "org.opensearch.test:framework:${opensearch_version}"
 
-    compileOnly group: 'org.apache.commons', name: 'commons-text', version: '1.10.0'
-    compileOnly group: 'com.google.code.gson', name: 'gson', version: '2.13.2'
+    compileOnly group: 'org.opensearch', name:'opensearch-security-spi', version:"${opensearch_build}"
+
+    compileOnly group: 'org.apache.commons', name: 'commons-text', version: '1.14.0'
+    compileOnly group: 'com.google.code.gson', name: 'gson', version: "${versions.gson}"
     compileOnly group: 'org.json', name: 'json', version: '20231013'
     testImplementation group: 'org.json', name: 'json', version: '20231013'
     implementation('com.google.guava:guava:32.1.3-jre') {
@@ -43,6 +45,10 @@ dependencies {
     compileOnly group: 'com.networknt' , name: 'json-schema-validator', version: '1.4.0'
     // Multi-tenant SDK Client
     compileOnly "org.opensearch:opensearch-remote-metadata-sdk:${opensearch_build}"
+    compileOnly (group: 'software.amazon.awssdk', name: 'netty-nio-client', version: "${versions.aws}") {
+        exclude(group: 'org.reactivestreams', module: 'reactive-streams')
+        exclude(group: 'org.slf4j', module: 'slf4j-api')
+    }
 }
 
 lombok {
@@ -107,10 +113,11 @@ publishing {
         }
         maven {
             name = "Snapshots" //  optional target repository name
-            url = "https://central.sonatype.com/repository/maven-snapshots/"
-            credentials {
-                username "$System.env.SONATYPE_USERNAME"
-                password "$System.env.SONATYPE_PASSWORD"
+            url = System.getenv("MAVEN_SNAPSHOTS_S3_REPO")
+            credentials(AwsCredentials) {
+                accessKey = System.getenv("AWS_ACCESS_KEY_ID")
+                secretKey = System.getenv("AWS_SECRET_ACCESS_KEY")
+                sessionToken = System.getenv("AWS_SESSION_TOKEN")
             }
         }
     }

common/src/main/java/org/opensearch/ml/common/CommonValue.java

@@ -46,7 +46,9 @@ public class CommonValue {
     public static final String ML_AGENT_INDEX = ".plugins-ml-agent";
     public static final String ML_MEMORY_META_INDEX = ".plugins-ml-memory-meta";
     public static final String ML_MEMORY_MESSAGE_INDEX = ".plugins-ml-memory-message";
-    public static final String ML_MEMORY_CONTAINER_INDEX = ".plugins-ml-memory-container";
+    public static final String ML_MEMORY_CONTAINER_INDEX = ".plugins-ml-am-memory-container";
+    public static final String ML_AGENTIC_MEMORY_SYSTEM_INDEX_PREFIX = ".plugins-ml-am";
+    public static final String ML_AGENTIC_MEMORY_INDEX_PATTERN = ML_AGENTIC_MEMORY_SYSTEM_INDEX_PREFIX + "*";
     public static final String ML_STOP_WORDS_INDEX = ".plugins-ml-stop-words";
     // index used in 2.19 to track MlTaskBatchUpdate task
     public static final String TASK_POLLING_JOB_INDEX = ".ml_commons_task_polling_job";
@@ -68,12 +70,19 @@ public class CommonValue {
     public static final String ML_MEMORY_META_INDEX_MAPPING_PATH = "index-mappings/ml_memory_meta.json";
     public static final String ML_MEMORY_MESSAGE_INDEX_MAPPING_PATH = "index-mappings/ml_memory_message.json";
     public static final String ML_MEMORY_CONTAINER_INDEX_MAPPING_PATH = "index-mappings/ml_memory_container.json";
+    public static final String ML_MEMORY_SESSION_INDEX_MAPPING_PATH = "index-mappings/ml_memory_sessions.json";
+    public static final String ML_WORKING_MEMORY_INDEX_MAPPING_PATH = "index-mappings/ml_memory_working.json";
+    public static final String ML_LONG_TERM_MEMORY_INDEX_MAPPING_PATH = "index-mappings/ml_memory_long_term.json";
+    public static final String ML_LONG_MEMORY_HISTORY_INDEX_MAPPING_PATH = "index-mappings/ml_memory_long_term_history.json";
     public static final String ML_MCP_SESSION_MANAGEMENT_INDEX_MAPPING_PATH = "index-mappings/ml_mcp_session_management.json";
     public static final String ML_MCP_TOOLS_INDEX_MAPPING_PATH = "index-mappings/ml_mcp_tools.json";
     public static final String ML_JOBS_INDEX_MAPPING_PATH = "index-mappings/ml_jobs.json";
     public static final String ML_INDEX_INSIGHT_CONFIG_INDEX_MAPPING_PATH = "index-mappings/ml_index_insight_config.json";
     public static final String ML_INDEX_INSIGHT_STORAGE_INDEX_MAPPING_PATH = "index-mappings/ml_index_insight_storage.json";
 
+    // Resource type used in resource-access-control
+    public static final String ML_MODEL_GROUP_RESOURCE_TYPE = "ml-model-group";
+
     // Calculate Versions independently of OpenSearch core version
     public static final Version VERSION_2_11_0 = Version.fromString("2.11.0");
     public static final Version VERSION_2_12_0 = Version.fromString("2.12.0");
@@ -86,6 +95,8 @@ public class CommonValue {
     public static final Version VERSION_2_19_0 = Version.fromString("2.19.0");
     public static final Version VERSION_3_0_0 = Version.fromString("3.0.0");
     public static final Version VERSION_3_1_0 = Version.fromString("3.1.0");
+    public static final Version VERSION_3_2_0 = Version.fromString("3.2.0");
+    public static final Version VERSION_3_3_0 = Version.fromString("3.3.0");
 
     // Connector Constants
     public static final String NAME_FIELD = "name";
@@ -112,7 +123,9 @@ public class CommonValue {
     public static final String MCP_CONNECTORS_FIELD = "mcp_connectors";
     public static final String MCP_CONNECTOR_ID_FIELD = "mcp_connector_id";
     public static final String MCP_DEFAULT_SSE_ENDPOINT = "/sse";
-    public static final String SSE_ENDPOINT_FILED = "sse_endpoint";
+    public static final String SSE_ENDPOINT_FIELD = "sse_endpoint";
+    public static final String MCP_DEFAULT_STREAMABLE_HTTP_ENDPOINT = "/mcp/";
+    public static final String ENDPOINT_FIELD = "endpoint";
 
     // TOOL Constants
     public static final String TOOL_INPUT_SCHEMA_FIELD = "input_schema";
@@ -121,4 +134,16 @@ public class CommonValue {
     public static final long INDEX_INSIGHT_GENERATING_TIMEOUT = 3 * 60 * 1000;
     public static final long INDEX_INSIGHT_UPDATE_INTERVAL = 24 * 60 * 60 * 1000;
 
+    // JSON-RPC Error Codes
+    public static final int JSON_RPC_PARSE_ERROR = -32700;
+    public static final int JSON_RPC_INTERNAL_ERROR = -32603;
+    public static final int JSON_RPC_SERVER_NOT_READY_ERROR = -32000;
+
+    // MCP Server response fields
+    public static final String ACKNOWLEDGE_FIELD = "acknowledged";
+    public static final String MCP_RESPONSE_FIELD = "mcp_response";
+    public static final String ERROR_FIELD = "error";
+    public static final String MESSAGE_FIELD = "message";
+    public static final String ID_FIELD = "id";
+    public static final String ERROR_CODE_FIELD = "error_code";
 }

common/src/main/java/org/opensearch/ml/common/MLModel.java

@@ -863,17 +863,17 @@ public Tags getTags() {
     public Tags getTags(Connector connector) {
         // if connector is present, model is a remote model
         if (this.algorithm == FunctionName.REMOTE && connector != null) {
-            return getRemoteModelTags(connector);
+            return getRemoteModelTags(connector).addTag(IS_HIDDEN_FIELD, isHidden != null && isHidden);
         }
 
         // pre-trained models follow a specific naming convention, relying on that to identify a pre-trained model
         if (this.name != null
             && (this.name.startsWith("amazon/") || this.name.startsWith("huggingface/"))
             && this.name.split("/").length >= 3) {
-            return getPreTrainedModelTags();
+            return getPreTrainedModelTags().addTag(IS_HIDDEN_FIELD, isHidden != null && isHidden);
         }
 
-        return getCustomModelTags();
+        return getCustomModelTags().addTag(IS_HIDDEN_FIELD, isHidden != null && isHidden);
     }
 
     /**

common/src/main/java/org/opensearch/ml/common/MLModelGroup.java

@@ -6,6 +6,7 @@
 package org.opensearch.ml.common;
 
 import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.opensearch.ml.common.CommonValue.BACKEND_ROLES_FIELD;
 import static org.opensearch.ml.common.CommonValue.TENANT_ID_FIELD;
 import static org.opensearch.ml.common.CommonValue.VERSION_2_19_0;
 
@@ -33,7 +34,6 @@ public class MLModelGroup implements ToXContentObject {
     public static final String MODEL_GROUP_NAME_FIELD = "name"; // name of the model group
     public static final String DESCRIPTION_FIELD = "description"; // description of the model group
     public static final String LATEST_VERSION_FIELD = "latest_version"; // latest model version added to the model group
-    public static final String BACKEND_ROLES_FIELD = "backend_roles"; // back_end roles as specified by the owner/admin
     public static final String OWNER = "owner"; // user who creates/owns the model group
 
     public static final String ACCESS = "access"; // assigned to public, private, or null when model group created