diff --git a/Dockerfile b/Dockerfile index 6bfc74a..5655dab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,10 @@ -FROM docker.elastic.co/logstash/logstash-oss:7.1.1 +FROM docker.elastic.co/logstash/logstash-oss:7.16.0 + +#ENV DEBUG=1 ENV pravega_client_auth_method=Bearer ENV pravega_client_auth_loadDynamic=true COPY . /pravega -RUN bin/logstash-plugin install /pravega/logstash-output-pravega-0.6.0.gem +RUN echo "OpenSSL::SSL.const_set(:VERIFY_PEER, OpenSSL::SSL::VERIFY_NONE)" >> lib/pluginmanager/install.rb +RUN bin/logstash-plugin install /pravega/logstash-output-pravega-0.10.1.gem diff --git a/README.md b/README.md index d7c46ce..6789bdc 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,8 @@ docker build -t ${DOCKER_REPOSITORY}/logstash-pravega:0.6.0 . docker push ${DOCKER_REPOSITORY}/logstash-pravega:0.6.0 ``` - Deploy it to K8S using helm install +edit charts/logstash/values.yaml for SDP cluster pravega controller, scope and stream +and also configure logstash conf for inputs,filters and output ``` helm upgrade --install logstash-pravega \ --namespace mynamespace charts/logstash/ \ diff --git a/charts/logstash/templates/statefulset.yaml b/charts/logstash/templates/statefulset.yaml index 13583bf..ab4f7b8 100644 --- a/charts/logstash/templates/statefulset.yaml +++ b/charts/logstash/templates/statefulset.yaml @@ -30,6 +30,7 @@ spec: checksum/patterns: {{ include (print $.Template.BasePath "/patterns-config.yaml") . | sha256sum }} checksum/templates: {{ include (print $.Template.BasePath "/files-config.yaml") . | sha256sum }} checksum/pipeline: {{ include (print $.Template.BasePath "/pipeline-config.yaml") . | sha256sum }} + nautilus.dellemc.com/pravega-environment: "{{ .Release.Namespace }}-pravega" {{- if .Values.podAnnotations }} ## Custom pod annotations {{- range $key, $value := .Values.podAnnotations }} @@ -141,7 +142,7 @@ spec: tolerations: {{ toYaml . | indent 8 }} {{- end }} - serviceAccountName: {{ template "logstash.serviceAccountName" . }} + #serviceAccountName: {{ template "logstash.serviceAccountName" . }} terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} volumes: - name: patterns diff --git a/charts/logstash/values.yaml b/charts/logstash/values.yaml index 6ed2b37..8a441ed 100644 --- a/charts/logstash/values.yaml +++ b/charts/logstash/values.yaml @@ -1,4 +1,4 @@ -replicaCount: 1 +replicaCount: 2 podDisruptionBudget: maxUnavailable: 1 @@ -15,7 +15,7 @@ image: # - name: "myKubernetesSecret" service: - type: ClusterIP + type: LoadBalancer # clusterIP: None # nodePort: # Set this to local, to preserve client source ip. Default stripes out the source ip @@ -31,13 +31,13 @@ service: # targetPort: syslog-udp # protocol: UDP syslog-tcp: - port: 1514 - targetPort: syslog-tcp - protocol: TCP - #beats: - # port: 5044 - # targetPort: beats - # protocol: TCP + port: 1514 + targetPort: syslog-tcp + protocol: TCP + #beats: + #port: 5044 + #targetPort: beats + #protocol: TCP # http: # port: 8080 # targetPort: http @@ -46,15 +46,15 @@ service: # loadBalancerSourceRanges: # - 192.168.0.1 ports: - # - name: syslog-udp - # containerPort: 1514 - # protocol: UDP - - name: syslog-tcp - containerPort: 1514 - protocol: TCP - #- name: beats - #containerPort: 5044 - #protocol: TCP + #- name: syslog-udp + #containerPort: 1514 + #protocol: UDP + - name: syslog-tcp + containerPort: 1514 + protocol: TCP + #- name: beats + #containerPort: 5044 + #protocol: TCP # - name: http # containerPort: 8080 # protocol: TCP @@ -115,7 +115,9 @@ podLabels: {} # team: "developers" # service: "logstash" -extraEnv: [] +extraEnv: + - name: JAVA_HOME + value: "/usr/share/logstash/jdk" extraInitContainers: [] # - name: echo @@ -131,7 +133,7 @@ livenessProbe: httpGet: path: / port: monitor - initialDelaySeconds: 20 + initialDelaySeconds: 40 # periodSeconds: 30 # timeoutSeconds: 30 # failureThreshold: 6 @@ -141,7 +143,7 @@ readinessProbe: httpGet: path: / port: monitor - initialDelaySeconds: 20 + initialDelaySeconds: 40 # periodSeconds: 30 # timeoutSeconds: 30 # failureThreshold: 6 @@ -156,7 +158,7 @@ persistence: ## set, choosing the default provisioner. (gp2 on AWS, standard on ## GKE, AWS & OpenStack) ## - # storageClass: "-" + storageClass: "nfs" accessMode: ReadWriteOnce size: 2Gi @@ -289,12 +291,14 @@ inputs: # type => syslog # } tcp { - port => 1514 - type => syslog + port => 1514 + type => syslog } #beats { # port => 5044 #} + # + # http { # port => 8080 # } @@ -309,20 +313,29 @@ inputs: } filters: - # main: |- - # filter { - # } + main: |- + filter { + grok { + match => { "message" => "%{TIMESTAMP_ISO8601:timestamp} %{DATA:request_id} %{IP:ecs_node}:%{NUMBER:ecs_node_port} %{IP:client}:%{NUMBER:client_port} ((?[^-]{1}[^\s]*)|-) ((?[^-]{1}[^\s]*)|-) %{WORD:method} ((?[^-]{1}[^\s]*)|-) ((?[^-]{1}[^\s]*)|-) ((?[^-]{1}[^\s]*)|-) ((?[^-]{1}[^\s]*)|-) HTTP/%{BASE16FLOAT:http_version} %{NUMBER:response_code:int} %{NUMBER:time-taken:int} (%{NUMBER:upload-size:int}|-) (%{NUMBER:download-size:int}|-) %{NUMBER:storage_processing:int}" } + } + + if "_grokparsefailure" in [tags] { + drop { } + } + } outputs: main: |- output { + stdout { codec => rubydebug } pravega { - pravega_endpoint => "tcp://localhost:9090" - stream_name => "" - scope => "" + pravega_endpoint => "tls://pravega-controller.sdp.sdp-ecs.log-analytics.com:443" + stream_name => "test-app-stream" + scope => "test-app" + create_scope => false codec => json { charset => "UTF-8" } } - # stdout { codec => rubydebug } + #stdout { codec => rubydebug } # elasticsearch { # hosts => ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}"] # manage_template => false @@ -336,10 +349,10 @@ outputs: # topic_id => "destination" # } } - + serviceAccount: # Specifies whether a ServiceAccount should be created - create: false + create: true` # The name of the ServiceAccount to use. # If not set and create is true, a name is generated using the fullname template name: serviceaccount diff --git a/logstash-output-pravega-0.10.1.gem b/logstash-output-pravega-0.10.1.gem new file mode 100644 index 0000000..38e08f5 Binary files /dev/null and b/logstash-output-pravega-0.10.1.gem differ diff --git a/logstash-output-pravega.gemspec b/logstash-output-pravega.gemspec index 317c2a5..822cc69 100644 --- a/logstash-output-pravega.gemspec +++ b/logstash-output-pravega.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'logstash-output-pravega' - s.version = '0.9.0' + s.version = '0.10.1' s.licenses = ['Apache License (2.0)'] s.summary = 'Output events to a Pravega Stream. This uses the Pravega Writer API to write event to a stream on the Pravega' s.description = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program' @@ -16,9 +16,9 @@ Gem::Specification.new do |s| # Special flag to let us know this is actually a logstash plugin s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" } - s.requirements << "jar 'io.pravega:pravega-client', '0.9.0'" + s.requirements << "jar 'io.pravega:pravega-client', '0.10.1'" - s.requirements << "jar 'io.pravega:pravega-keycloak-client', '0.9.0'" + s.requirements << "jar 'io.pravega:pravega-keycloak-client', '0.10.1'" # Gem dependencies s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" diff --git a/pom.xml b/pom.xml index 2bb164a..079db25 100644 --- a/pom.xml +++ b/pom.xml @@ -5,8 +5,8 @@ dummy 0.0.0 - 0.6.0 - 0.6.0 + 0.10.1 + 0.10.1 @@ -36,13 +36,13 @@ io.pravega pravega-client - 0.9.0 + ${pravegaVersion} io.pravega pravega-keycloak-client - 0.9.0 + ${keycloak-credentials-version}