Elastic

elastic/README.md

@@ -0,0 +1,40 @@
+`minikube start --nodes=3`
+`minikube addons enable ingress`
+
+
+https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-deploy-eck.html
+
+Install custom resource definitions and the operator with its RBAC rules:
+
+`kubectl apply -f https://download.elastic.co/downloads/eck/1.5.0/all-in-one.yaml`
+
+Monitor the operator logs:
+
+`kubectl -n elastic-system logs -f statefulset.apps/elastic-operator`
+
+`kubectl get elasticsearch`
+
+`kubectl get service quickstart-es-http`
+
+`PASSWORD=$(kubectl get secret elasticsearch-es-elastic-user -o go-template='{{.data.elastic | base64decode}}') && echo $PASSWORD`
+
+kubectl port-forward -n default service/kibana-kb-http 5601
+kubectl port-forward -n default service/logstash 5044
+kubectl port-forward service/quickstart-es-http 9200
+kubectl port-forward service/quickstart-es-http 9200
+
+kubectl expose service kibana-kb-http --port=5601 --target-port=5601 --name=kibana-web --type=NodePort
+kubectl expose service tb-web-ui --port=5601 --target-port=5601 --name=kibana-web --type=NodePort
+
+minikube service kibana-web --url
+
+curl -u "elastic:$PASSWORD" -k "https://quickstart-es-http:9200"
+
+elastic
+25Q9l7zOsT64i6B6oV1mX9CD
+
+FILEBEAT
+
+https://www.elastic.co/guide/en/beats/filebeat/7.12/running-on-kubernetes.html
+
+

elastic/docker/README.md

@@ -0,0 +1,8 @@
+docker run -d \
+--name=filebeat \
+--restart unless-stopped \
+--user=root \
+--network="host" \
+--volume="$(pwd)/elastic/docker/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro" \
+--volume="/var/log/splunk/:/var/log/splunk/:ro" \
+docker.elastic.co/beats/filebeat:7.12.0 filebeat -e -strict.perms=false

elastic/docker/filebeat.docker.yml

@@ -0,0 +1,26 @@
+filebeat.config:
+  modules:
+    path: ${path.config}/modules.d/*.yml
+    reload.enabled: false
+filebeat.inputs:
+  - type: log
+    tags: ["splunk"]
+#    json.keys_under_root: true
+#    json.add_error_key: true
+#      json.message_key: log
+    paths:
+      - /var/log/splunk/*.json
+#filebeat.autodiscover:
+#  providers:
+#    - type: docker
+#      hints.enabled: true
+#processors:
+#- add_cloud_metadata: ~
+#output.elasticsearch:
+#  hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:5044}'
+#    username: '${ELASTICSEARCH_USERNAME:elastic}'
+#    password: '${ELASTICSEARCH_PASSWORD:25Q9l7zOsT64i6B6oV1mX9CD}'
+output.logstash:
+  hosts: [ "127.0.0.1:5044" ]
+#  username: '${ELASTICSEARCH_USERNAME:elastic}'
+#  password: '${ELASTICSEARCH_PASSWORD:25Q9l7zOsT64i6B6oV1mX9CD}'

elastic/elasticsearch.yaml

@@ -0,0 +1,21 @@
+apiVersion: elasticsearch.k8s.elastic.co/v1
+kind: Elasticsearch
+metadata:
+  name: elasticsearch
+spec:
+  version: 7.12.0
+  nodeSets:
+    - name: default
+      count: 2
+      volumeClaimTemplates:
+        - metadata:
+            name: elasticsearch-data
+          spec:
+            accessModes:
+              - ReadWriteOnce
+            resources:
+              requests:
+                storage: 10Gi
+            storageClassName: standard
+      config:
+        node.store.allow_mmap: false

elastic/filebeat-kubernetes.yaml

@@ -0,0 +1,171 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: filebeat-config
+  namespace: kube-system
+  labels:
+    k8s-app: filebeat
+data:
+  filebeat.yml: |-
+    filebeat.inputs:
+    - type: container
+      paths:
+        - /var/log/containers/*.log
+      processors:
+        - add_kubernetes_metadata:
+            host: ${NODE_NAME}
+            matchers:
+            - logs_path:
+                logs_path: "/var/log/containers/"
+
+    # To enable hints based autodiscover, remove `filebeat.inputs` configuration and uncomment this:
+    #filebeat.autodiscover:
+    #  providers:
+    #    - type: kubernetes
+    #      node: ${NODE_NAME}
+    #      hints.enabled: true
+    #      hints.default_config:
+    #        type: container
+    #        paths:
+    #          - /var/log/containers/*${data.kubernetes.container.id}.log
+
+    processors:
+      - add_cloud_metadata:
+      - add_host_metadata:
+
+    cloud.id: ${ELASTIC_CLOUD_ID}
+    cloud.auth: ${ELASTIC_CLOUD_AUTH}
+
+    output.elasticsearch:
+      hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}']
+      username: ${ELASTICSEARCH_USERNAME}
+      password: ${ELASTICSEARCH_PASSWORD}
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: filebeat
+  namespace: kube-system
+  labels:
+    k8s-app: filebeat
+spec:
+  selector:
+    matchLabels:
+      k8s-app: filebeat
+  template:
+    metadata:
+      labels:
+        k8s-app: filebeat
+    spec:
+      serviceAccountName: filebeat
+      terminationGracePeriodSeconds: 30
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      containers:
+      - name: filebeat
+        image: docker.elastic.co/beats/filebeat:7.12.0
+        args: [
+          "-c", "/etc/filebeat.yml",
+          "-e",
+        ]
+        env:
+        - name: ELASTICSEARCH_HOST
+          value: quickstart-es-http
+        - name: ELASTICSEARCH_PORT
+          value: "9200"
+        - name: ELASTICSEARCH_USERNAME
+          value: elastic
+        - name: ELASTICSEARCH_PASSWORD
+          value: 0479LnfD3oejDzE7P092ks6O
+        - name: ELASTIC_CLOUD_ID
+          value:
+        - name: ELASTIC_CLOUD_AUTH
+          value:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        securityContext:
+          runAsUser: 0
+          # If using Red Hat OpenShift uncomment this:
+          #privileged: true
+        resources:
+          limits:
+            memory: 200Mi
+          requests:
+            cpu: 100m
+            memory: 100Mi
+        volumeMounts:
+        - name: config
+          mountPath: /etc/filebeat.yml
+          readOnly: true
+          subPath: filebeat.yml
+        - name: data
+          mountPath: /usr/share/filebeat/data
+        - name: varlibdockercontainers
+          mountPath: /var/lib/docker/containers
+          readOnly: true
+        - name: varlog
+          mountPath: /var/log
+          readOnly: true
+      volumes:
+      - name: config
+        configMap:
+          defaultMode: 0640
+          name: filebeat-config
+      - name: varlibdockercontainers
+        hostPath:
+          path: /var/lib/docker/containers
+      - name: varlog
+        hostPath:
+          path: /var/log
+      # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart
+      - name: data
+        hostPath:
+          # When filebeat runs as non-root user, this directory needs to be writable by group (g+w).
+          path: /var/lib/filebeat-data
+          type: DirectoryOrCreate
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: filebeat
+subjects:
+- kind: ServiceAccount
+  name: filebeat
+  namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: filebeat
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: filebeat
+  labels:
+    k8s-app: filebeat
+rules:
+- apiGroups: [""] # "" indicates the core API group
+  resources:
+  - namespaces
+  - pods
+  - nodes
+  verbs:
+  - get
+  - watch
+  - list
+- apiGroups: ["apps"]
+  resources:
+    - replicasets
+  verbs: ["get", "list", "watch"]
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: filebeat
+  namespace: kube-system
+  labels:
+    k8s-app: filebeat
+---

elastic/filebeat_autodiscover.yaml

@@ -0,0 +1,120 @@
+apiVersion: beat.k8s.elastic.co/v1beta1
+kind: Beat
+metadata:
+  name: filebeat
+spec:
+  type: filebeat
+  version: 7.12.0
+#  elasticsearchRef:
+#    name: elasticsearch
+#  kibanaRef:
+#    name: kibana
+  config:
+    output.logstash:
+      hosts: [ "logstash:5044" ]
+    filebeat.autodiscover:
+        providers:
+        - type: kubernetes
+          node: ${NODE_NAME}
+          hints:
+            enabled: true
+            default_config:
+              type: container
+              paths:
+              - /var/log/containers/*${data.kubernetes.container.id}.log
+    processors:
+    - add_cloud_metadata: {}
+    - add_host_metadata: {}
+  daemonSet:
+    podTemplate:
+      spec:
+        serviceAccountName: filebeat
+        automountServiceAccountToken: true
+        terminationGracePeriodSeconds: 30
+        dnsPolicy: ClusterFirstWithHostNet
+        hostNetwork: true # Allows to provide richer host metadata
+        containers:
+        - name: filebeat
+          securityContext:
+            runAsUser: 0
+            # If using Red Hat OpenShift uncomment this:
+            #privileged: true
+          volumeMounts:
+          - name: varlogcontainers
+            mountPath: /var/log/containers
+          - name: varlogpods
+            mountPath: /var/log/pods
+          - name: varlibdockercontainers
+            mountPath: /var/lib/docker/containers
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+        volumes:
+        - name: varlogcontainers
+          hostPath:
+            path: /var/log/containers
+        - name: varlogpods
+          hostPath:
+            path: /var/log/pods
+        - name: varlibdockercontainers
+          hostPath:
+            path: /var/lib/docker/containers
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: filebeat
+rules:
+- apiGroups: [""] # "" indicates the core API group
+  resources:
+  - namespaces
+  - pods
+  - nodes
+  verbs:
+  - get
+  - watch
+  - list
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: filebeat
+  namespace: thingsboard
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: filebeat
+subjects:
+- kind: ServiceAccount
+  name: filebeat
+#  namespace: default
+roleRef:
+  kind: ClusterRole
+  name: filebeat
+  apiGroup: rbac.authorization.k8s.io
+#---
+#apiVersion: elasticsearch.k8s.elastic.co/v1
+#kind: Elasticsearch
+#metadata:
+#  name: elasticsearch
+#spec:
+#  version: 7.12.0
+#  nodeSets:
+#  - name: default
+#    count: 3
+#    config:
+#      node.store.allow_mmap: false
+#---
+#apiVersion: kibana.k8s.elastic.co/v1
+#kind: Kibana
+#metadata:
+#  name: kibana
+#spec:
+#  version: 7.12.0
+#  count: 1
+#  elasticsearchRef:
+#    name: elasticsearch
+...