From 65a19e13f58873c4d289fc474940327191962ba6 Mon Sep 17 00:00:00 2001 From: Yerzhaisang Taskali Date: Thu, 6 Feb 2025 17:06:32 -0500 Subject: [PATCH 1/4] Add instructions to run OpenSearch from source code Signed-off-by: Yerzhaisang Taskali --- DEVELOPER_GUIDE.md | 32 +++++++++++ dev-docker-compose.yml | 125 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 157 insertions(+) create mode 100644 dev-docker-compose.yml diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 865c23d7bd..d6c6fda997 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -82,6 +82,38 @@ opensearch.username: "admin" # Default username opensearch.password: "admin" # Default password ``` +### 🚀 Run OpenSearch from Source Code + +#### 1. Build the Distributions +First, compile the source code to build the distributions: + +```bash +./gradlew assemble +``` + +After the build completes, you'll find an artifact like `opensearch-ml-3.0.0.0-SNAPSHOT.zip` in the `plugin/build/distributions` directory. + +--- + +#### 2. Run OpenSearch Using Docker Compose +Start OpenSearch with: + +```bash +docker-compose -f dev-docker-compose.yml up -d +``` + +To stop OpenSearch, run: + +```bash +docker-compose -f dev-docker-compose.yml down +``` + +--- + +#### 3. Access OpenSearch +After a few minutes, the server will be ready. You can make requests using `curl` or other HTTP clients + + ### Build This package uses the [Gradle](https://docs.gradle.org/current/userguide/userguide.html) build system. Gradle comes with excellent documentation that should be your first stop when trying to figure out how to operate or modify the build. we also use the OpenSearch build tools for Gradle. These tools are idiosyncratic and don't always follow the conventions and instructions for building regular Java code using Gradle. Not everything in this package will work the way it's described in the Gradle documentation. If you encounter such a situation, the OpenSearch build tools [source code](https://github.com/opensearch-project/OpenSearch/tree/main/buildSrc/src/main/groovy/org/opensearch/gradle) is your best bet for figuring out what's going on. diff --git a/dev-docker-compose.yml b/dev-docker-compose.yml new file mode 100644 index 0000000000..176d2948ce --- /dev/null +++ b/dev-docker-compose.yml @@ -0,0 +1,125 @@ +version: '3' +services: + opensearch-node1: + image: opensearchstaging/opensearch:3.0.0 + container_name: opensearch-node1 + environment: + - cluster.name=opensearch-cluster + - node.name=opensearch-node1 + - discovery.seed_hosts=opensearch-node1,opensearch-node2 + - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2 + - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=MyPassword123! + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - "DISABLE_SECURITY_PLUGIN=true" + + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems + hard: 65536 + volumes: + - opensearch-data1:/usr/share/opensearch/data + - ./plugin/build/distributions:/usr/share/opensearch/dev + command: bash -c "bin/opensearch-plugin remove opensearch-skills; bin/opensearch-plugin remove opensearch-ml; bin/opensearch-plugin install --batch file:///usr/share/opensearch/dev/opensearch-ml-3.0.0.0-SNAPSHOT.zip; ./opensearch-docker-entrypoint.sh opensearch" + ports: + - 9200:9200 + - 9600:9600 # required for Performance Analyzer + - 5005:5005 + + networks: + - opensearch-net + opensearch-node2: + image: opensearchstaging/opensearch:3.0.0 + container_name: opensearch-node2 + environment: + - cluster.name=opensearch-cluster + - node.name=opensearch-node2 + - discovery.seed_hosts=opensearch-node1,opensearch-node2 + - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2 + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=MyPassword123! + - "DISABLE_SECURITY_PLUGIN=true" + + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + volumes: + - opensearch-data2:/usr/share/opensearch/data + - ./plugin/build/distributions:/usr/share/opensearch/dev + command: bash -c "bin/opensearch-plugin remove opensearch-skills; bin/opensearch-plugin remove opensearch-ml; bin/opensearch-plugin install --batch file:///usr/share/opensearch/dev/opensearch-ml-3.0.0.0-SNAPSHOT.zip; ./opensearch-docker-entrypoint.sh opensearch" + networks: + - opensearch-net + + opensearch-ml1: + image: opensearchstaging/opensearch:3.0.0 + container_name: opensearch-ml1 + environment: + - cluster.name=opensearch-cluster + - node.name=opensearch-ml1 + - node.roles=ml + - discovery.seed_hosts=opensearch-node1,opensearch-node2 + - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2 + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=MyPassword123! + - "DISABLE_SECURITY_PLUGIN=true" + + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + volumes: + - opensearch-ml1:/usr/share/opensearch/data + - ./plugin/build/distributions:/usr/share/opensearch/dev + command: bash -c "bin/opensearch-plugin remove opensearch-skills; bin/opensearch-plugin remove opensearch-ml; bin/opensearch-plugin install --batch file:///usr/share/opensearch/dev/opensearch-ml-3.0.0.0-SNAPSHOT.zip; ./opensearch-docker-entrypoint.sh opensearch" + networks: + - opensearch-net + + opensearch-ml2: + image: opensearchstaging/opensearch:3.0.0 + container_name: opensearch-ml2 + environment: + - cluster.name=opensearch-cluster + - node.name=opensearch-ml2 + - node.roles=ml + - discovery.seed_hosts=opensearch-node1,opensearch-node2 + - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2 + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=MyPassword123! + - "DISABLE_SECURITY_PLUGIN=true" + + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + volumes: + - opensearch-ml2:/usr/share/opensearch/data + - ./plugin/build/distributions:/usr/share/opensearch/dev + command: bash -c "bin/opensearch-plugin remove opensearch-skills; bin/opensearch-plugin remove opensearch-ml; bin/opensearch-plugin install --batch file:///usr/share/opensearch/dev/opensearch-ml-3.0.0.0-SNAPSHOT.zip; ./opensearch-docker-entrypoint.sh opensearch" + networks: + - opensearch-net + +volumes: + opensearch-data1: + opensearch-data2: + opensearch-ml1: + opensearch-ml2: + + +networks: + opensearch-net: From aa9a2b0a330a4699512f2a54b71fe3850b13cf25 Mon Sep 17 00:00:00 2001 From: Yerzhaisang Taskali Date: Tue, 11 Feb 2025 17:24:53 -0500 Subject: [PATCH 2/4] change based on reviewer feedback Signed-off-by: Yerzhaisang Taskali --- DEVELOPER_GUIDE.md | 2 +- .../docker/dev-docker-compose.yml | 40 +++++++++++++++++-- 2 files changed, 37 insertions(+), 5 deletions(-) rename dev-docker-compose.yml => docs/docker/dev-docker-compose.yml (63%) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index d6c6fda997..12b64a924f 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -82,7 +82,7 @@ opensearch.username: "admin" # Default username opensearch.password: "admin" # Default password ``` -### 🚀 Run OpenSearch from Source Code +### 🚀 Run OpenSearch from Source Code using Docker #### 1. Build the Distributions First, compile the source code to build the distributions: diff --git a/dev-docker-compose.yml b/docs/docker/dev-docker-compose.yml similarity index 63% rename from dev-docker-compose.yml rename to docs/docker/dev-docker-compose.yml index 176d2948ce..2db860780d 100644 --- a/dev-docker-compose.yml +++ b/docs/docker/dev-docker-compose.yml @@ -21,8 +21,16 @@ services: soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems hard: 65536 volumes: + # Persist data for the first OpenSearch node to ensure cluster state, indexes, and metadata are retained - opensearch-data1:/usr/share/opensearch/data - - ./plugin/build/distributions:/usr/share/opensearch/dev + + # Mount the local plugin build directory into the container to make custom plugins available for installation + - ../../plugin/build/distributions:/usr/share/opensearch/dev + + # We will now remove the existing 'opensearch-skills' plugin to prevent version conflicts + # Remove the existing 'opensearch-ml' plugin to ensure a clean installation + # Install the new version of the 'opensearch-ml' plugin from a local distribution file + # Start the OpenSearch service using the Docker entrypoint script command: bash -c "bin/opensearch-plugin remove opensearch-skills; bin/opensearch-plugin remove opensearch-ml; bin/opensearch-plugin install --batch file:///usr/share/opensearch/dev/opensearch-ml-3.0.0.0-SNAPSHOT.zip; ./opensearch-docker-entrypoint.sh opensearch" ports: - 9200:9200 @@ -52,8 +60,16 @@ services: soft: 65536 hard: 65536 volumes: + # Persist data for the second OpenSearch node to maintain high availability and redundancy - opensearch-data2:/usr/share/opensearch/data - - ./plugin/build/distributions:/usr/share/opensearch/dev + + # Mount the local plugin build directory into the container to make custom plugins available for installation + - ../../plugin/build/distributions:/usr/share/opensearch/dev + + # We will now remove the existing 'opensearch-skills' plugin to prevent version conflicts + # Remove the existing 'opensearch-ml' plugin to ensure a clean installation + # Install the new version of the 'opensearch-ml' plugin from a local distribution file + # Start the OpenSearch service using the Docker entrypoint script command: bash -c "bin/opensearch-plugin remove opensearch-skills; bin/opensearch-plugin remove opensearch-ml; bin/opensearch-plugin install --batch file:///usr/share/opensearch/dev/opensearch-ml-3.0.0.0-SNAPSHOT.zip; ./opensearch-docker-entrypoint.sh opensearch" networks: - opensearch-net @@ -80,8 +96,16 @@ services: soft: 65536 hard: 65536 volumes: + # Persist machine learning-related data for the first ML node to store model metadata and results - opensearch-ml1:/usr/share/opensearch/data - - ./plugin/build/distributions:/usr/share/opensearch/dev + + # Mount the local plugin build directory into the container to make custom plugins available for installation + - ../../plugin/build/distributions:/usr/share/opensearch/dev + + # We will now remove the existing 'opensearch-skills' plugin to prevent version conflicts + # Remove the existing 'opensearch-ml' plugin to ensure a clean installation + # Install the new version of the 'opensearch-ml' plugin from a local distribution file + # Start the OpenSearch service using the Docker entrypoint script command: bash -c "bin/opensearch-plugin remove opensearch-skills; bin/opensearch-plugin remove opensearch-ml; bin/opensearch-plugin install --batch file:///usr/share/opensearch/dev/opensearch-ml-3.0.0.0-SNAPSHOT.zip; ./opensearch-docker-entrypoint.sh opensearch" networks: - opensearch-net @@ -108,8 +132,16 @@ services: soft: 65536 hard: 65536 volumes: + # Persist machine learning-related data for the second ML node to support distributed ML processing and data redundancy - opensearch-ml2:/usr/share/opensearch/data - - ./plugin/build/distributions:/usr/share/opensearch/dev + + # Mount the local plugin build directory into the container to make custom plugins available for installation + - ../../plugin/build/distributions:/usr/share/opensearch/dev + + # We will now remove the existing 'opensearch-skills' plugin to prevent version conflicts + # Remove the existing 'opensearch-ml' plugin to ensure a clean installation + # Install the new version of the 'opensearch-ml' plugin from a local distribution file + # Start the OpenSearch service using the Docker entrypoint script command: bash -c "bin/opensearch-plugin remove opensearch-skills; bin/opensearch-plugin remove opensearch-ml; bin/opensearch-plugin install --batch file:///usr/share/opensearch/dev/opensearch-ml-3.0.0.0-SNAPSHOT.zip; ./opensearch-docker-entrypoint.sh opensearch" networks: - opensearch-net From c656bd291d200cf537209d29b208476880f23e06 Mon Sep 17 00:00:00 2001 From: Yerzhaisang Taskali Date: Tue, 11 Feb 2025 17:39:42 -0500 Subject: [PATCH 3/4] change based on reviewer feedback Signed-off-by: Yerzhaisang Taskali --- DEVELOPER_GUIDE.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 12b64a924f..8f58c960b8 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -96,16 +96,10 @@ After the build completes, you'll find an artifact like `opensearch-ml-3.0.0.0-S --- #### 2. Run OpenSearch Using Docker Compose -Start OpenSearch with: +Navigate to the root directory of the project and start OpenSearch with the following command: ```bash -docker-compose -f dev-docker-compose.yml up -d -``` - -To stop OpenSearch, run: - -```bash -docker-compose -f dev-docker-compose.yml down +docker-compose -f docs/docker/dev-docker-compose.yml up ``` --- From b0d7ec049c40dd433c170629fa6fc7dab68384ec Mon Sep 17 00:00:00 2001 From: Yerzhaisang Taskali Date: Tue, 18 Feb 2025 14:38:49 -0500 Subject: [PATCH 4/4] add example request and some comments Signed-off-by: Yerzhaisang Taskali --- DEVELOPER_GUIDE.md | 10 +++++----- docs/docker/dev-docker-compose.yml | 24 ++++++++++++------------ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 8f58c960b8..147f1d4f8c 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -82,7 +82,7 @@ opensearch.username: "admin" # Default username opensearch.password: "admin" # Default password ``` -### 🚀 Run OpenSearch from Source Code using Docker +### Run OpenSearch from Source Code using Docker #### 1. Build the Distributions First, compile the source code to build the distributions: @@ -93,7 +93,6 @@ First, compile the source code to build the distributions: After the build completes, you'll find an artifact like `opensearch-ml-3.0.0.0-SNAPSHOT.zip` in the `plugin/build/distributions` directory. ---- #### 2. Run OpenSearch Using Docker Compose Navigate to the root directory of the project and start OpenSearch with the following command: @@ -102,11 +101,12 @@ Navigate to the root directory of the project and start OpenSearch with the foll docker-compose -f docs/docker/dev-docker-compose.yml up ``` ---- #### 3. Access OpenSearch -After a few minutes, the server will be ready. You can make requests using `curl` or other HTTP clients - +After a few minutes, the server will be ready. You can make requests using `curl` or other HTTP clients. For example, +```bash +curl -X GET http://localhost:9200/ +``` ### Build diff --git a/docs/docker/dev-docker-compose.yml b/docs/docker/dev-docker-compose.yml index 2db860780d..bef9896f89 100644 --- a/docs/docker/dev-docker-compose.yml +++ b/docs/docker/dev-docker-compose.yml @@ -4,18 +4,18 @@ services: image: opensearchstaging/opensearch:3.0.0 container_name: opensearch-node1 environment: - - cluster.name=opensearch-cluster - - node.name=opensearch-node1 - - discovery.seed_hosts=opensearch-node1,opensearch-node2 - - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2 + - cluster.name=opensearch-cluster # Sets the name of the OpenSearch cluster + - node.name=opensearch-node1 # Sets the name of the node in the cluster + - discovery.seed_hosts=opensearch-node1,opensearch-node2 # Defines initial nodes for cluster discovery + - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2 # Defines initial cluster manager nodes - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping - - OPENSEARCH_INITIAL_ADMIN_PASSWORD=MyPassword123! - - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" - - "DISABLE_SECURITY_PLUGIN=true" + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=MyPassword123! # Sets the initial admin password + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # Sets JVM heap size to 512MB + - "DISABLE_SECURITY_PLUGIN=true" # Disables the security plugin for easy access ulimits: memlock: - soft: -1 + soft: -1 # Ensures unlimited memory locking hard: -1 nofile: soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems @@ -33,12 +33,12 @@ services: # Start the OpenSearch service using the Docker entrypoint script command: bash -c "bin/opensearch-plugin remove opensearch-skills; bin/opensearch-plugin remove opensearch-ml; bin/opensearch-plugin install --batch file:///usr/share/opensearch/dev/opensearch-ml-3.0.0.0-SNAPSHOT.zip; ./opensearch-docker-entrypoint.sh opensearch" ports: - - 9200:9200 + - 9200:9200 # Main OpenSearch API port - 9600:9600 # required for Performance Analyzer - - 5005:5005 + - 5005:5005 # Debugging port networks: - - opensearch-net + - opensearch-net # Connects to OpenSearch network opensearch-node2: image: opensearchstaging/opensearch:3.0.0 container_name: opensearch-node2 @@ -80,7 +80,7 @@ services: environment: - cluster.name=opensearch-cluster - node.name=opensearch-ml1 - - node.roles=ml + - node.roles=ml # Defines this node as an ML node - discovery.seed_hosts=opensearch-node1,opensearch-node2 - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2 - bootstrap.memory_lock=true