Support SDP 1.3

Dockerfile

@@ -1,7 +1,10 @@
-FROM docker.elastic.co/logstash/logstash-oss:7.1.1
+FROM docker.elastic.co/logstash/logstash-oss:7.16.0
+
+#ENV DEBUG=1
 
 ENV pravega_client_auth_method=Bearer
 ENV pravega_client_auth_loadDynamic=true
 COPY . /pravega
-RUN bin/logstash-plugin install /pravega/logstash-output-pravega-0.6.0.gem
+RUN   echo "OpenSSL::SSL.const_set(:VERIFY_PEER, OpenSSL::SSL::VERIFY_NONE)" >> lib/pluginmanager/install.rb
+RUN bin/logstash-plugin install  /pravega/logstash-output-pravega-0.10.1.gem
 

README.md

@@ -156,6 +156,8 @@ docker build -t ${DOCKER_REPOSITORY}/logstash-pravega:0.6.0 .
 docker push  ${DOCKER_REPOSITORY}/logstash-pravega:0.6.0
 ```
 - Deploy it to K8S using helm install
+edit charts/logstash/values.yaml for SDP cluster pravega controller, scope and stream
+and also configure logstash conf for inputs,filters and output
 ```
 helm upgrade --install logstash-pravega \
      --namespace mynamespace  charts/logstash/ \

charts/logstash/templates/statefulset.yaml

@@ -30,6 +30,7 @@ spec:
         checksum/patterns: {{ include (print $.Template.BasePath "/patterns-config.yaml") . | sha256sum }}
         checksum/templates: {{ include (print $.Template.BasePath "/files-config.yaml") . | sha256sum }}
         checksum/pipeline: {{ include (print $.Template.BasePath "/pipeline-config.yaml") . | sha256sum }}
+        nautilus.dellemc.com/pravega-environment: "{{ .Release.Namespace }}-pravega"
       {{- if .Values.podAnnotations }}
         ## Custom pod annotations
         {{- range $key, $value := .Values.podAnnotations }}
@@ -141,7 +142,7 @@ spec:
       tolerations:
 {{ toYaml . | indent 8 }}
     {{- end }}
-      serviceAccountName: {{ template "logstash.serviceAccountName" . }}
+    #serviceAccountName: {{ template "logstash.serviceAccountName" . }}
       terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }}
       volumes:
         - name: patterns

charts/logstash/values.yaml

@@ -1,4 +1,4 @@
-replicaCount: 1
+replicaCount: 2
 
 podDisruptionBudget:
   maxUnavailable: 1
@@ -15,7 +15,7 @@ image:
   #    - name: "myKubernetesSecret"
 
 service:
-  type: ClusterIP
+  type: LoadBalancer
   # clusterIP: None
   # nodePort:
   # Set this to local, to preserve client source ip.  Default stripes out the source ip
@@ -31,13 +31,13 @@ service:
     #   targetPort: syslog-udp
     #   protocol: UDP
     syslog-tcp:
-       port: 1514
-       targetPort: syslog-tcp
-       protocol: TCP
-       #beats:
-       #  port: 5044
-       #  targetPort: beats
-       # protocol: TCP
+      port: 1514
+      targetPort: syslog-tcp
+      protocol: TCP
+      #beats:
+      #port: 5044
+      #targetPort: beats
+      #protocol: TCP
     # http:
     #  port: 8080
     #  targetPort: http
@@ -46,15 +46,15 @@ service:
     # loadBalancerSourceRanges:
     #   - 192.168.0.1
 ports:
-  # - name: syslog-udp
-  #   containerPort: 1514
-  #   protocol: UDP
-  - name: syslog-tcp
-    containerPort: 1514
-    protocol: TCP
-    #- name: beats
-    #containerPort: 5044
-    #protocol: TCP
+        #- name: syslog-udp
+        #containerPort: 1514
+        #protocol: UDP
+    - name: syslog-tcp
+      containerPort: 1514
+      protocol: TCP
+        #- name: beats
+        #containerPort: 5044
+        #protocol: TCP
   # - name: http
   #   containerPort: 8080
   #   protocol: TCP
@@ -115,7 +115,9 @@ podLabels: {}
   # team: "developers"
   # service: "logstash"
 
-extraEnv: []
+extraEnv:
+  - name: JAVA_HOME
+    value: "/usr/share/logstash/jdk"
 
 extraInitContainers: []
   # - name: echo
@@ -131,7 +133,7 @@ livenessProbe:
   httpGet:
     path: /
     port: monitor
-  initialDelaySeconds: 20
+  initialDelaySeconds: 40
   # periodSeconds: 30
   # timeoutSeconds: 30
   # failureThreshold: 6
@@ -141,7 +143,7 @@ readinessProbe:
   httpGet:
     path: /
     port: monitor
-  initialDelaySeconds: 20
+  initialDelaySeconds: 40
   # periodSeconds: 30
   # timeoutSeconds: 30
   # failureThreshold: 6
@@ -156,7 +158,7 @@ persistence:
   ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
   ##   GKE, AWS & OpenStack)
   ##
-  # storageClass: "-"
+  storageClass: "nfs"
   accessMode: ReadWriteOnce
   size: 2Gi
 
@@ -289,12 +291,14 @@ inputs:
       #   type => syslog
       # }
       tcp {
-        port => 1514
-        type => syslog
+       port => 1514
+       type => syslog
       }
       #beats {
       #  port => 5044
       #}
+      #
+
       # http {
       #   port => 8080
       # }
@@ -309,20 +313,29 @@ inputs:
     }
 
 filters:
-  # main: |-
-  #   filter {
-  #   }
+   main: |-
+     filter {
+       grok {
+          match => { "message" => "%{TIMESTAMP_ISO8601:timestamp} %{DATA:request_id} %{IP:ecs_node}:%{NUMBER:ecs_node_port} %{IP:client}:%{NUMBER:client_port} ((?<user>[^-]{1}[^\s]*)|-) ((?<uri>[^-]{1}[^\s]*)|-) %{WORD:method} ((?<namespace>[^-]{1}[^\s]*)|-) ((?<bucket>[^-]{1}[^\s]*)|-) ((?<key>[^-]{1}[^\s]*)|-) ((?<query>[^-]{1}[^\s]*)|-) HTTP/%{BASE16FLOAT:http_version} %{NUMBER:response_code:int} %{NUMBER:time-taken:int} (%{NUMBER:upload-size:int}|-) (%{NUMBER:download-size:int}|-) %{NUMBER:storage_processing:int}" }
+        }
+
+        if "_grokparsefailure" in [tags] {
+               drop { }
+          }
+     }
 
 outputs:
   main: |-
     output {
+      stdout { codec => rubydebug }
       pravega {
-        pravega_endpoint => "tcp://localhost:9090"
-        stream_name => "<changeme>"
-        scope => "<changeme>"
+        pravega_endpoint => "tls://pravega-controller.sdp.sdp-ecs.log-analytics.com:443"
+        stream_name => "test-app-stream"
+        scope => "test-app"
+        create_scope => false
         codec => json { charset => "UTF-8" }
         }
-      # stdout { codec => rubydebug }
+        #stdout { codec => rubydebug }
       # elasticsearch {
       #  hosts => ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}"]
       #  manage_template => false
@@ -336,10 +349,10 @@ outputs:
       #   topic_id => "destination"
       # }
     }
-
+    
 serviceAccount:
   # Specifies whether a ServiceAccount should be created
-  create: false
+  create: true`
   # The name of the ServiceAccount to use.
   # If not set and create is true, a name is generated using the fullname template
   name: serviceaccount

logstash-output-pravega.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-pravega'
-  s.version       = '0.9.0'
+  s.version       = '0.10.1'
   s.licenses      = ['Apache License (2.0)']
   s.summary       = 'Output events to a Pravega Stream. This uses the Pravega Writer API to write event to a stream on the Pravega'
   s.description   = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program'
@@ -16,9 +16,9 @@ Gem::Specification.new do |s|
 
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
-  s.requirements << "jar 'io.pravega:pravega-client', '0.9.0'"
+  s.requirements << "jar 'io.pravega:pravega-client', '0.10.1'"
 
-  s.requirements << "jar 'io.pravega:pravega-keycloak-client', '0.9.0'"
+  s.requirements << "jar 'io.pravega:pravega-keycloak-client', '0.10.1'"
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"

pom.xml

@@ -5,8 +5,8 @@
    <artifactId>dummy</artifactId>
    <version>0.0.0</version>
    <properties>
-     <keycloak-credentials-version>0.6.0</keycloak-credentials-version>
-     <pravegaVersion>0.6.0</pravegaVersion>
+     <keycloak-credentials-version>0.10.1</keycloak-credentials-version>
+     <pravegaVersion>0.10.1</pravegaVersion>
    </properties>
    <repositories>
       <repository>
@@ -36,13 +36,13 @@
     <dependency>
       <groupId>io.pravega</groupId>
       <artifactId>pravega-client</artifactId>
-      <version>0.9.0</version>
+      <version>${pravegaVersion}</version>
     </dependency>
 
     <dependency>
       <groupId>io.pravega</groupId>
       <artifactId>pravega-keycloak-client</artifactId>
-      <version>0.9.0</version>
+      <version>${keycloak-credentials-version}</version>
     </dependency>
   </dependencies>
 </project>