feat: Elasticsearch log provider module

.github/workflows/e2e.yml

@@ -10,6 +10,7 @@ on:
       - 'strimzi-mcp/**'
       - 'systemtest/**'
       - 'loki-log-provider/**'
+      - 'elasticsearch-log-provider/**'
       - 'metrics-prometheus/**'
       - 'dev/**'
       - '!**/*.md'
@@ -53,9 +54,12 @@ jobs:
     - name: Deploy Loki and Promtail
       run: ./dev/scripts/setup-loki-kind.sh deploy
 
+    - name: Deploy Elasticsearch and Fluent Bit
+      run: ./dev/scripts/setup-elasticsearch-kind.sh deploy
+
     - name: Build MCP image and load into kind
       run: |
-        ./mvnw package -pl common,metrics-prometheus,loki-log-provider,strimzi-mcp \
+        ./mvnw package -pl common,metrics-prometheus,loki-log-provider,elasticsearch-log-provider,strimzi-mcp \
           -DskipTests \
           -Dquarkus.container-image.build=true \
           -Dquarkus.container-image.tag=test

AGENTS.md

@@ -51,6 +51,7 @@ Documentation updates are mandatory and must be completed before the task is con
 - **`common`** (`streamshub-mcp-common`) - Generic Kubernetes helpers, DTOs, MCP framework utilities shared across modules
 - **`metrics-prometheus`** (`streamshub-metrics-prometheus`) - Prometheus/Thanos metrics provider (pluggable, replaceable JAR)
 - **`loki-log-provider`** (`streamshub-loki-log-provider`) - Loki log provider (pluggable, replaceable JAR)
+- **`elasticsearch-log-provider`** (`streamshub-elasticsearch-log-provider`) - Elasticsearch/OpenSearch log provider (pluggable, replaceable JAR)
 - **`strimzi-mcp`** (`strimzi-mcp`) - MCP Server for Strimzi: Kafka management tools and services
 
 ## Build & Test
@@ -132,6 +133,17 @@ io.streamshub.mcp.loki.
 └── LokiLogProvider (implements LogCollectorProvider)
 ```
 
+### Elasticsearch log provider module (`elasticsearch-log-provider/`)
+
+```
+io.streamshub.mcp.elasticsearch.
+├── config/    → ElasticsearchConfig (field mapping, auth mode, index pattern)
+├── service/   → ElasticsearchClient, ElasticsearchAuthFilter
+├── util/      → ElasticsearchQuerySanitizer
+├── ElasticsearchResponse (response record with nested Hit record)
+└── ElasticsearchLogProvider (implements LogCollectorProvider)
+```
+
 ### MCP Server for Strimzi module (`strimzi-mcp/`)
 
 ```

CHANGELOG.md

@@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- **Elasticsearch/OpenSearch log provider** (`mcp.log.provider=streamshub-elasticsearch`) for querying logs from Elasticsearch or OpenSearch
+- **Dev environment manifests and scripts** for deploying Elasticsearch with Fluent Bit on Kind clusters and Elastic search operator on Openshift
+- **System tests** for Elasticsearch log provider covering log collection, field mapping, time window queries, and error handling
+- **Thanos and VictoriaMetrics compatibility documentation** — the existing Prometheus metrics provider works with Thanos Querier and VictoriaMetrics without code changes
+
 ### Changed
 
 ### Fixed

README.md

@@ -4,6 +4,7 @@
 [![Release](https://img.shields.io/github/v/release/streamshub/streamshub-mcp)](https://github.com/streamshub/streamshub-mcp/releases/latest)
 [![License](https://img.shields.io/github/license/streamshub/streamshub-mcp)](LICENSE)
 ![Java](https://img.shields.io/badge/Java-21%2B-blue)
+
 StreamsHub MCP provides Model Context Protocol (MCP) servers that give AI assistants direct access to Kubernetes-based streaming infrastructure.
 Each component of your streaming stack gets its own MCP server that can be deployed independently.
 

dev/manifests/elasticsearch-kind/elasticsearch.yaml

@@ -0,0 +1,64 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: elasticsearch
+  namespace: elasticsearch
+  labels:
+    app: elasticsearch
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: elasticsearch
+  template:
+    metadata:
+      labels:
+        app: elasticsearch
+    spec:
+      containers:
+        - name: elasticsearch
+          image: docker.elastic.co/elasticsearch/elasticsearch:9.5.1
+          env:
+            - name: discovery.type
+              value: single-node
+            - name: xpack.security.enabled
+              value: "false"
+            - name: ES_JAVA_OPTS
+              value: "-Xms256m -Xmx256m"
+          ports:
+            - name: http
+              containerPort: 9200
+          readinessProbe:
+            httpGet:
+              path: /_cluster/health?wait_for_status=yellow
+              port: http
+            initialDelaySeconds: 30
+            periodSeconds: 10
+          resources:
+            requests:
+              memory: 512Mi
+              cpu: 250m
+            limits:
+              memory: 1Gi
+          volumeMounts:
+            - name: data
+              mountPath: /usr/share/elasticsearch/data
+      volumes:
+        - name: data
+          emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: elasticsearch
+  namespace: elasticsearch
+  labels:
+    app: elasticsearch
+spec:
+  type: ClusterIP
+  ports:
+    - name: http
+      port: 9200
+      targetPort: http
+  selector:
+    app: elasticsearch

dev/manifests/elasticsearch-kind/fluentbit-config.yaml

@@ -0,0 +1,51 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: fluent-bit-config
+  namespace: elasticsearch
+data:
+  fluent-bit.conf: |
+    [SERVICE]
+        Flush        1
+        Log_Level    info
+        Parsers_File parsers.conf
+
+    [INPUT]
+        Name             tail
+        Path             /var/log/containers/*.log
+        Parser           cri
+        Tag              kube.*
+        Refresh_Interval 5
+        Mem_Buf_Limit    5MB
+        Skip_Long_Lines  On
+
+    [FILTER]
+        Name                kubernetes
+        Match               kube.*
+        Kube_URL            https://kubernetes.default.svc:443
+        Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
+        Merge_Log           On
+        Keep_Log            Off
+        K8S-Logging.Parser  On
+
+    [OUTPUT]
+        Name            es
+        Match           kube.*
+        Host            elasticsearch.elasticsearch.svc.cluster.local
+        Port            9200
+        Index           kubernetes
+        Type            _doc
+        Suppress_Type_Name On
+        Logstash_Format On
+        Logstash_Prefix kubernetes
+        Time_Key        @timestamp
+        Replace_Dots    On
+
+  parsers.conf: |
+    [PARSER]
+        Name        cri
+        Format      regex
+        Regex       ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$
+        Time_Key    time
+        Time_Format %Y-%m-%dT%H:%M:%S.%L%z

dev/manifests/elasticsearch-kind/fluentbit.yaml

@@ -0,0 +1,67 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: fluent-bit
+  namespace: elasticsearch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: fluent-bit
+rules:
+  - apiGroups: [""]
+    resources: ["namespaces", "pods", "pods/log"]
+    verbs: ["get", "watch", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: fluent-bit
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: fluent-bit
+subjects:
+  - kind: ServiceAccount
+    name: fluent-bit
+    namespace: elasticsearch
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: fluent-bit
+  namespace: elasticsearch
+  labels:
+    app: fluent-bit
+spec:
+  selector:
+    matchLabels:
+      app: fluent-bit
+  template:
+    metadata:
+      labels:
+        app: fluent-bit
+    spec:
+      serviceAccountName: fluent-bit
+      containers:
+        - name: fluent-bit
+          image: fluent/fluent-bit:3.2
+          resources:
+            requests:
+              memory: 64Mi
+              cpu: 50m
+            limits:
+              memory: 256Mi
+          volumeMounts:
+            - name: config
+              mountPath: /fluent-bit/etc
+            - name: varlog
+              mountPath: /var/log
+              readOnly: true
+      volumes:
+        - name: config
+          configMap:
+            name: fluent-bit-config
+        - name: varlog
+          hostPath:
+            path: /var/log

dev/manifests/elasticsearch-kind/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - elasticsearch.yaml
+  - fluentbit-config.yaml
+  - fluentbit.yaml

dev/manifests/elasticsearch-kind/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: elasticsearch

dev/manifests/elasticsearch/cluster-logging.yaml

@@ -0,0 +1,96 @@
+---
+apiVersion: operators.coreos.com/v1
+kind: OperatorGroup
+metadata:
+  name: elasticsearch-logging
+  namespace: elasticsearch-logging
+spec:
+  targetNamespaces:
+    - elasticsearch-logging
+---
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  name: cluster-logging
+  namespace: elasticsearch-logging
+spec:
+  channel: stable-6.6
+  name: cluster-logging
+  source: redhat-operators
+  sourceNamespace: openshift-marketplace
+  installPlanApproval: Automatic
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: logcollector
+  namespace: elasticsearch-logging
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: logcollector-es-collect-application
+subjects:
+  - kind: ServiceAccount
+    name: logcollector
+    namespace: elasticsearch-logging
+roleRef:
+  kind: ClusterRole
+  name: collect-application-logs
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: logcollector-es-collect-infrastructure
+subjects:
+  - kind: ServiceAccount
+    name: logcollector
+    namespace: elasticsearch-logging
+roleRef:
+  kind: ClusterRole
+  name: collect-infrastructure-logs
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: observability.openshift.io/v1
+kind: ClusterLogForwarder
+metadata:
+  name: strimzi-logs-elasticsearch
+  namespace: elasticsearch-logging
+spec:
+  serviceAccount:
+    name: logcollector
+  outputs:
+    - name: elasticsearch-output
+      type: elasticsearch
+      elasticsearch:
+        url: https://elasticsearch-es-http.elasticsearch-logging.svc:9200
+        index:
+          name: kubernetes
+        authentication:
+          username:
+            secretName: elasticsearch-es-credentials
+            key: username
+          password:
+            secretName: elasticsearch-es-credentials
+            key: password
+      tls:
+        ca:
+          secretName: elasticsearch-es-http-certs-public
+          key: ca.crt
+  inputs:
+    - name: strimzi-app-logs
+      type: application
+      application:
+        includes:
+          - namespace: strimzi-kafka
+          - namespace: strimzi
+          - namespace: strimzi-drain-cleaner
+          - namespace: strimzi-kafka-mirror
+          - namespace: strimzi-mirror-maker
+  pipelines:
+    - name: strimzi-to-elasticsearch
+      inputRefs:
+        - strimzi-app-logs
+      outputRefs:
+        - elasticsearch-output

dev/manifests/elasticsearch/eck-operator.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  name: elastic-cloud-eck
+  namespace: openshift-operators
+spec:
+  channel: stable
+  name: elastic-cloud-eck
+  source: certified-operators
+  sourceNamespace: openshift-marketplace
+  installPlanApproval: Automatic