diff --git a/docs/mint.json b/docs/mint.json index 625116c134..a98da8666f 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -219,6 +219,7 @@ "providers/documentation/ollama-provider", "providers/documentation/openai-provider", "providers/documentation/openobserve-provider", + "providers/documentation/opensearch-provider", "providers/documentation/opensearchserverless-provider", "providers/documentation/openshift-provider", "providers/documentation/opsgenie-provider", diff --git a/docs/providers/documentation/opensearch-provider.mdx b/docs/providers/documentation/opensearch-provider.mdx new file mode 100644 index 0000000000..c99ac71629 --- /dev/null +++ b/docs/providers/documentation/opensearch-provider.mdx @@ -0,0 +1,65 @@ +--- +title: "OpenSearch" +sidebarTitle: "OpenSearch Provider" +description: "OpenSearch provider enables Keep to query and write to self-managed OpenSearch clusters." +--- + +## Overview + +The OpenSearch provider connects Keep to a self-managed OpenSearch cluster. You can run queries, index documents, and surface alerts from OpenSearch monitors into Keep. + +### Key Features +- Query indices with the OpenSearch Query DSL. +- Index documents (with optional document IDs and refresh policy). +- TLS and basic authentication support; custom CA bundle (`ca_certs`) for private PKI. + +## Prerequisites +- An accessible OpenSearch endpoint (host and port). +- If authentication is enabled, a user with query/write permissions to the target indices. +- For HTTPS with private CAs, a PEM bundle path for certificate verification. + +## Authentication Configuration +- **host** (Required): OpenSearch host (e.g., `opensearch.example.com` or `http://localhost`) +- **port** (Required): OpenSearch port (default `9200`) +- **username/password** (Optional): Set when using Basic auth (both must be provided together) +- **use_ssl** (Optional): Enable HTTPS (default: true) +- **verify_certs** (Optional): Verify server certificates (default: true) +- **ca_certs** (Optional): PEM bundle path for custom CAs when using HTTPS + +## Querying OpenSearch +Use the `_query` action to search an index. + +- **index**: Target index name. +- **query**: OpenSearch query DSL object (stringified JSON also supported). +- **size** (optional): Limit the number of returned documents. + +Example: +```json +{ + "index": "keep" + "query": { "match_all": {} }, + "size": 1 +} +``` + +## Writing to OpenSearch +Use the `_notify` action to index documents. + +- **index**: Target index name. +- **document**: JSON object (or stringified JSON) to index. +- **doc_id** (optional): Explicit document ID. +- **refresh** (optional): Refresh policy (`true`, `false`, or `"wait_for"`). + +Example: +```json +{ + "index": "keep", + "document": { "message": "Keep test doc" }, + "doc_id": "doc_1", + "refresh": true +} +``` + +## Useful Links +- [OpenSearch Query DSL](https://opensearch.org/docs/latest/query-dsl/index/) +- [Index API](https://opensearch.org/docs/latest/api-reference/document-apis/index-document/) diff --git a/docs/providers/overview.mdx b/docs/providers/overview.mdx index 37d08694e6..062da42f00 100644 --- a/docs/providers/overview.mdx +++ b/docs/providers/overview.mdx @@ -596,6 +596,14 @@ By leveraging Keep Providers, users are able to deeply integrate Keep with the t } > + + } +> + + Retrieves all the documents from index "keep", + and uploads a document to OSS OpenSearch in index "keep". + disabled: false + + triggers: + - type: manual + + steps: + - name: query-index + provider: + type: opensearch + config: "{{ providers.opensearch }}" + with: + index: keep + query: + match_all: {} + size: 1 + + actions: + - name: create-doc + provider: + type: opensearch + config: "{{ providers.opensearch }}" + with: + index: keep + document: + message: Keep test doc + doc_id: doc_1 + refresh: true \ No newline at end of file diff --git a/keep-ui/public/icons/opensearch-icon.png b/keep-ui/public/icons/opensearch-icon.png new file mode 100644 index 0000000000..77ea00b50d Binary files /dev/null and b/keep-ui/public/icons/opensearch-icon.png differ diff --git a/keep/providers/opensearch_provider/__init__.py b/keep/providers/opensearch_provider/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/keep/providers/opensearch_provider/opensearch_provider.py b/keep/providers/opensearch_provider/opensearch_provider.py new file mode 100644 index 0000000000..85c2d11557 --- /dev/null +++ b/keep/providers/opensearch_provider/opensearch_provider.py @@ -0,0 +1,198 @@ +import dataclasses +import json +import typing + +import pydantic +from opensearchpy import OpenSearch + +from keep.contextmanager.contextmanager import ContextManager +from keep.exceptions.provider_connection_failed import ProviderConnectionFailed +from keep.exceptions.provider_exception import ProviderException +from keep.providers.base.base_provider import BaseProvider +from keep.providers.models.provider_config import ProviderConfig, ProviderScope + + +@pydantic.dataclasses.dataclass +class OpensearchProviderAuthConfig: + host: str = dataclasses.field( + default=None, + metadata={ + "required": True, + "description": "OpenSearch host", + "sensitive": False, + }, + ) + port: str = dataclasses.field( + default=9200, + metadata={ + "required": True, + "description": "OpenSearch port", + "sensitive": False, + }, + ) + username: typing.Optional[str] = dataclasses.field( + default=None, + metadata={ + "description": "OpenSearch username", + "config_sub_group": "username_password", + "config_main_group": "authentication", + "sensitive": False, + }, + ) + password: typing.Optional[str] = dataclasses.field( + default=None, + metadata={ + "description": "OpenSearch password", + "config_sub_group": "username_password", + "config_main_group": "authentication", + "sensitive": True, + }, + ) + use_ssl: bool = dataclasses.field( + default=True, + metadata={ + "description": "Enable SSL", + "type": "switch", + "config_main_group": "authentication", + }, + ) + verify_certs: bool = dataclasses.field( + default=True, + metadata={ + "description": "Enable SSL certificate verification", + "type": "switch", + "config_main_group": "authentication", + }, + ) + ca_certs: typing.Optional[str] = dataclasses.field( + default=None, + metadata={ + "description": "CA bundle path (PEM) for HTTPS connections", + "config_main_group": "authentication", + "config_sub_group": "ssl", + "placeholder": "/path/to/ca.pem", + }, + ) + + +class OpensearchProvider(BaseProvider): + PROVIDER_DISPLAY_NAME = "OpenSearch" + PROVIDER_CATEGORY = ["Database", "Observability"] + + PROVIDER_SCOPES = [ + ProviderScope( + name="connect_to_server", + description="The user can connect to the server", + mandatory=True, + alias="Connect to the server", + ) + ] + + def __init__( + self, context_manager: ContextManager, provider_id: str, config: ProviderConfig + ): + super().__init__(context_manager, provider_id, config) + self._client: OpenSearch | None = None + + def dispose(self): + pass + + def validate_config(self): + self.authentication_config = OpensearchProviderAuthConfig( + **self.config.authentication + ) + + def validate_scopes(self): + try: + self.client.ping() + scopes = { + "connect_to_server": True, + } + except Exception as e: + self.logger.exception("Error validating scopes") + scopes = { + "connect_to_server": str(e), + } + return scopes + + @property + def client(self) -> OpenSearch: + if not self._client: + self._client = self.__initialize_client() + return self._client + + def __initialize_client(self) -> OpenSearch: + scheme = "https" if self.authentication_config.use_ssl else "http" + auth = None + if self.authentication_config.username: + auth = ( + self.authentication_config.username, + self.authentication_config.password, + ) + client = OpenSearch( + hosts=[ + { + "host": self.authentication_config.host, + "port": self.authentication_config.port, + "scheme": scheme, + } + ], + http_auth=auth, + use_ssl=self.authentication_config.use_ssl, + verify_certs=self.authentication_config.verify_certs, + ca_certs=self.authentication_config.ca_certs, + ) + try: + client.info() + except Exception as e: + raise ProviderConnectionFailed(f"Failed to connect to OpenSearch: {str(e)}") + return client + + def _query(self, query: dict | str, index: str, size: int | None = None, **kwargs): + if not index: + raise ProviderException("Missing index for OpenSearch query") + body = query + if isinstance(body, str): + body = json.loads(body) + if size is not None: + if not isinstance(body, dict): + raise ProviderException("Query must be an object when specifying size") + body = dict(body) + body["size"] = size + response = self.client.search(index=index, body=body) + hits = response.get("hits", {}).get("hits") + return hits or [] + + def _notify( + self, + index: str, + document: dict | str, + doc_id: str | None = None, + refresh: bool | str | None = None, + **kwargs, + ): + if not index: + raise ProviderException("Missing index for OpenSearch document") + if document is None: + raise ProviderException("Missing document for OpenSearch") + body = document + if isinstance(body, str): + body = json.loads(body) + params: dict[str, typing.Any] = { + "index": index, + "body": body, + } + if doc_id: + params["id"] = doc_id + if refresh is not None: + params["refresh"] = refresh + return self.client.index(**params) + + def get_provider_metadata(self) -> dict: + try: + info = self.client.info() + version = info.get("version", {}).get("number") + return {"version": version} if version else {"version": "unknown"} + except Exception: + self.logger.exception("Failed to get OpenSearch metadata") + return {} diff --git a/poetry.lock b/poetry.lock index be82be4dd8..ea24c2160c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.0 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. [[package]] name = "aiofiles" @@ -1439,6 +1439,17 @@ pyarrow = ["pyarrow (>=1)"] requests = ["requests (>=2.4.0,!=2.32.2,<3.0.0)"] vectorstore-mmr = ["numpy (>=1)", "simsimd (>=3)"] +[[package]] +name = "events" +version = "0.5" +description = "Bringing the elegance of C# EventHandler to Python" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "Events-0.5-py3-none-any.whl", hash = "sha256:a7286af378ba3e46640ac9825156c93bdba7502174dd696090fdfcd4d80a1abd"}, +] + [[package]] name = "execnet" version = "2.1.1" @@ -3234,6 +3245,47 @@ typing-extensions = ">=4.7,<5" [package.extras] datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] +[[package]] +name = "opensearch-protobufs" +version = "0.19.0" +description = "" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "opensearch_protobufs-0.19.0-py3-none-any.whl", hash = "sha256:5137c9c2323cc7debb694754b820ca4cfb5fc8eb180c41ff125698c3ee11bfc2"}, +] + +[package.dependencies] +grpcio = ">=1.68.1" +protobuf = ">=3.20.3" + +[[package]] +name = "opensearch-py" +version = "3.1.0" +description = "Python client for OpenSearch" +optional = false +python-versions = "<4,>=3.10" +groups = ["main"] +files = [ + {file = "opensearch_py-3.1.0-py3-none-any.whl", hash = "sha256:e5af83d0454323e6ea9ddee8c0dcc185c0181054592d23cb701da46271a3b65b"}, + {file = "opensearch_py-3.1.0.tar.gz", hash = "sha256:883573af13175ff102b61c80b77934a9e937bdcc40cda2b92051ad53336bc055"}, +] + +[package.dependencies] +certifi = ">=2024.07.04" +Events = "*" +opensearch-protobufs = "0.19.0" +python-dateutil = "*" +requests = ">=2.32.0,<3.0.0" +urllib3 = {version = ">=1.26.19,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1,<3", markers = "python_version >= \"3.10\""} + +[package.extras] +async = ["aiohttp (>=3.12.14,<4)"] +develop = ["black (>=24.3.0)", "botocore", "coverage (<8.0.0)", "jinja2", "myst_parser", "pytest (>=3.0.0)", "pytest-cov", "pytest-mock (<4.0.0)", "pytz", "pyyaml", "requests (>=2.0.0,<3.0.0)", "sphinx", "sphinx_copybutton", "sphinx_rtd_theme"] +docs = ["aiohttp (>=3.12.14,<4)", "myst_parser", "sphinx", "sphinx_copybutton", "sphinx_rtd_theme"] +kerberos = ["requests_kerberos"] + [[package]] name = "openshift-client" version = "2.0.5" @@ -6107,4 +6159,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.14" -content-hash = "48583ec01b5b1cde6d02f4c6ea0a2827eab937b14ff21b62abc91602c400efde" +content-hash = "446fca5c54470e29f96bd4dbb9ccb637785f328c199c509e7817a4864221721c" diff --git a/pyproject.toml b/pyproject.toml index bfc101ad1d..d523a82467 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ google-cloud-secret-manager = "^2.16.1" sqlalchemy = "^2.0.14" snowflake-connector-python = "3.13.1" openai = "1.37.1" +opensearch-py = "3.1.0" opentelemetry-sdk = "1.29.0" opentelemetry-instrumentation-fastapi = "0.50b0" opentelemetry-instrumentation-logging = "0.50b0"