diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..6cb050d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,25 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +registries: + docker-registry-germanedge-docker-artifactory-new-solutions-com: + type: docker-registry + url: https://germanedge-docker.artifactory.new-solutions.com + username: dependabot + password: ${{secrets.ARTIFACTORY_DEPENDABOT_PASSWORD}} +updates: +- package-ecosystem: docker + directory: "/" + schedule: + interval: daily + open-pull-requests-limit: 10 + registries: + - docker-registry-germanedge-docker-artifactory-new-solutions-com +- package-ecosystem: gradle + directory: "/" + schedule: + interval: daily + open-pull-requests-limit: 10 diff --git a/Dockerfile b/Dockerfile index 2ac723f..b6fed07 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,34 +1,60 @@ -FROM wurstmeister/base +FROM germanedge-docker.artifactory.new-solutions.com/edge-one/ge-ubuntu-generic:2.5.0 -MAINTAINER Wurstmeister +ARG zookeper_version=3.8.1 -ENV ZOOKEEPER_VERSION 3.4.13 +ENV ZOOKEEPER_VERSION=$zookeper_version +ENV PORT=2181 +ENV SERVICENAME=zookeeper +ENV CONSUL_TAGS='"web","application","prometheus"' +ENV CONSUL_META_SCRAPE_PATH="\/metrics" +ENV CONSUL_META_SCRAPE_PORT="7071" +ENV FILEBEAT_ARGS='--E filebeat.inputs.2.paths=["/opt/zookeeper/logs/*.log"]' -#Download Zookeeper -RUN wget -q http://mirror.vorboss.net/apache/zookeeper/zookeeper-${ZOOKEEPER_VERSION}/zookeeper-${ZOOKEEPER_VERSION}.tar.gz && \ -wget -q https://www.apache.org/dist/zookeeper/KEYS && \ -wget -q https://www.apache.org/dist/zookeeper/zookeeper-${ZOOKEEPER_VERSION}/zookeeper-${ZOOKEEPER_VERSION}.tar.gz.asc && \ -wget -q https://www.apache.org/dist/zookeeper/zookeeper-${ZOOKEEPER_VERSION}/zookeeper-${ZOOKEEPER_VERSION}.tar.gz.md5 +USER root + +#RUN mkdir /var/run/sshd +#RUN echo 'root:germanedge' | chpasswd +#RUN apt update; apt upgrade; apt install -y wget unzip openjdk-8-jre-headless wget supervisor docker.io openssh-server curl +RUN apt update; apt upgrade; apt install -y wget unzip openjdk-8-jre-headless wget supervisor curl +#RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config +#RUN sed -i 's/^#\s*\(deb.*universe\)$/\1/g' /etc/apt/sources.list +RUN rm -rf /var/lib/apt/lists/* +#RUN echo '#!/bin/sh' > /usr/sbin/policy-rc.d && echo 'exit 101' >> /usr/sbin/policy-rc.d && chmod +x /usr/sbin/policy-rc.d && dpkg-divert --local --rename --add /sbin/initctl && cp -a /usr/sbin/policy-rc.d /sbin/initctl && sed -i 's/^exit.*/exit 0/' /sbin/initctl && echo 'force-unsafe-io' > /etc/dpkg/dpkg.cfg.d/docker-apt-speedup && echo 'DPkg::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' > /etc/apt/apt.conf.d/docker-clean && echo 'APT::Update::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' >> /etc/apt/apt.conf.d/docker-clean && echo 'Dir::Cache::pkgcache ""; Dir::Cache::srcpkgcache "";' >> /etc/apt/apt.conf.d/docker-clean && echo 'Acquire::Languages "none";' > /etc/apt/apt.conf.d/docker-no-languages && echo 'Acquire::GzipIndexes "true"; Acquire::CompressionTypes::Order:: "gz";' > /etc/apt/apt.conf.d/docker-gzip-indexes -#Verify download -RUN md5sum -c zookeeper-${ZOOKEEPER_VERSION}.tar.gz.md5 && \ -gpg --import KEYS && \ -gpg --verify zookeeper-${ZOOKEEPER_VERSION}.tar.gz.asc -#Install -RUN tar -xzf zookeeper-${ZOOKEEPER_VERSION}.tar.gz -C /opt +#Download Zookeeper +RUN curl https://downloads.apache.org/zookeeper/zookeeper-${ZOOKEEPER_VERSION}/apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz -o /tmp/zookeeper.tar.gz \ + && tar -xzf /tmp/zookeeper.tar.gz -C /opt \ + && mv /opt/apache-zookeeper-${ZOOKEEPER_VERSION}-bin /opt/zookeeper \ + && rm -rf /tmp/zookeeper.tar.gz #Configure -RUN mv /opt/zookeeper-${ZOOKEEPER_VERSION}/conf/zoo_sample.cfg /opt/zookeeper-${ZOOKEEPER_VERSION}/conf/zoo.cfg +RUN mv /opt/zookeeper/conf/zoo_sample.cfg /opt/zookeeper/conf/zoo.cfg -ENV JAVA_HOME /usr/lib/jvm/java-7-openjdk-amd64 -ENV ZK_HOME /opt/zookeeper-${ZOOKEEPER_VERSION} +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 +ENV ZK_HOME /opt/zookeeper RUN sed -i "s|/tmp/zookeeper|$ZK_HOME/data|g" $ZK_HOME/conf/zoo.cfg; mkdir $ZK_HOME/data -ADD start-zk.sh /usr/bin/start-zk.sh -EXPOSE 2181 2888 3888 +COPY --chown=edgeone:root start-zk.sh /usr/bin/start-zk.sh +#EXPOSE 2181 2888 3888 + + +RUN mkdir -p /opt/prometheus/ \ + && curl https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.17.2/jmx_prometheus_javaagent-0.17.2.jar -o /opt/prometheus/jmx-exporter.jar + +COPY --chown=edgeone:root prometheus_zk.yml /opt/prometheus/ + +ENV SERVER_JVMFLAGS='-javaagent:/opt/prometheus/jmx-exporter.jar=7071:/opt/prometheus/prometheus_zk.yml' + + +WORKDIR /opt/zookeeper +VOLUME ["/opt/zookeeper/conf", "/opt/zookeeper/data"] + +USER 1000 + +COPY --chown=edgeone:root startup.sh /app/startup.sh +COPY --chown=edgeone:root service.json /app/service.json -WORKDIR /opt/zookeeper-${ZOOKEEPER_VERSION} -VOLUME ["/opt/zookeeper-${ZOOKEEPER_VERSION}/conf", "/opt/zookeeper-${ZOOKEEPER_VERSION}/data"] +RUN chmod +x /app/startup.sh -CMD /usr/sbin/sshd && bash /usr/bin/start-zk.sh +USER root diff --git a/GitVersion.yml b/GitVersion.yml new file mode 100644 index 0000000..76268a4 --- /dev/null +++ b/GitVersion.yml @@ -0,0 +1,13 @@ +mode: Mainline +branches: + master: + is-mainline: false + regex: ^master$ + increment: none + develop: + is-mainline: true + regex: ^dev(elop)?(ment)?$ + tag: "" +ignore: + sha: [] +merge-message-formats: {} \ No newline at end of file diff --git a/README.md b/README.md index 9db3509..9e638d0 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,6 @@ zookeeper-docker ================ +Orchestrates kafka cluster + +Maintainer: Christoph Heuwieser + diff --git a/azure-pipelines-deploy.yml b/azure-pipelines-deploy.yml new file mode 100644 index 0000000..ba3ae21 --- /dev/null +++ b/azure-pipelines-deploy.yml @@ -0,0 +1,26 @@ +name: "zookeeper_deploy_$(Date:yyyyMMdd)$(Rev:.r)" + +trigger: none +pr: none + +resources: + repositories: + - repository: templates + type: github + name: Germanedge/azure-pipelines-templates + endpoint: Germanedge + pipelines: + - pipeline: zookeeper + source: edgeone/build/Germanedge.zookeeper + trigger: + branches: + include: + - develop + stages: + - Publish + +extends: + template: kubernetes-deployment-template.yaml@templates + parameters: + environment: edgeone-kubernetes.default + namespace: e1-dev diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 0000000..f8dadd2 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,24 @@ +# EdgeOne zookeeper Docker build pipeline +# Example: germanedge-docker/zookeeper:latest +# dockerRegistry: "artifactory.new-solutions.com" -> image container registry service + +variables: + - group: Github-credentials + - group: artifactory-docker + - name: imageName + value: "zookeeper" + +name: "$(imageName) $(GitVersion.FullSemVer)" + +resources: + repositories: + - repository: templates + type: github + name: Germanedge/azure-pipelines-templates + endpoint: Germanedge + +extends: + template: azure-pipelines-docker-template.yml@templates + parameters: + gitPassword: $(gitPassword) + gitUser: $(gitUser) diff --git a/manifests/deployment.yaml b/manifests/deployment.yaml new file mode 100644 index 0000000..cc4a09b --- /dev/null +++ b/manifests/deployment.yaml @@ -0,0 +1,56 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: zookeeper +spec: + selector: + matchLabels: + app: zookeeper + strategy: + type: Recreate + template: + metadata: + labels: + app: zookeeper + spec: + securityContext: + fsGroup: 1000 + volumes: + - name: edgeone-zookeeper-pv + persistentVolumeClaim: + claimName: edgeone-zookeeper-pv-claim + imagePullSecrets: + - name: artifactory + nodeSelector: + kubernetes.io/os: linux + containers: + - imagePullPolicy: Always + name: zookeeper + env: + - name: EXT_CA_CERT + valueFrom: + secretKeyRef: + name: external-ca-cert-secret + key: ext_ca_cert + optional: true + - name: K8S_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + command: [] + ports: + - containerPort: 2181 + name: port0 + protocol: TCP + - containerPort: 2181 + name: port1 + protocol: UDP + image: germanedge-docker.artifactory.new-solutions.com/edge-one/zookeeper:3.87.0 + resources: + limits: + memory: "512M" + volumeMounts: + - mountPath: /opt/zookeeper/data + name: edgeone-zookeeper-pv + subPath: optzookeeperdata +--- diff --git a/manifests/service.yaml b/manifests/service.yaml new file mode 100644 index 0000000..65061f5 --- /dev/null +++ b/manifests/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: zookeeper +spec: + selector: + app: zookeeper + type: ClusterIP + ports: + - targetPort: 2181 + port: 2181 + name: port0 +--- diff --git a/prometheus_zk.yml b/prometheus_zk.yml new file mode 100644 index 0000000..2a383cd --- /dev/null +++ b/prometheus_zk.yml @@ -0,0 +1,25 @@ +# source: https://raw.githubusercontent.com/prometheus/jmx_exporter/master/example_configs/zookeeper.yaml + +rules: + # replicated Zookeeper + - pattern: "org.apache.ZooKeeperService<>(\\w+)" + name: "zookeeper_$2" + - pattern: "org.apache.ZooKeeperService<>(\\w+)" + name: "zookeeper_$3" + labels: + replicaId: "$2" + - pattern: "org.apache.ZooKeeperService<>(\\w+)" + name: "zookeeper_$4" + labels: + replicaId: "$2" + memberType: "$3" + - pattern: "org.apache.ZooKeeperService<>(\\w+)" + name: "zookeeper_$4_$5" + labels: + replicaId: "$2" + memberType: "$3" + # standalone Zookeeper + - pattern: "org.apache.ZooKeeperService<>(\\w+)" + name: "zookeeper_$2" + - pattern: "org.apache.ZooKeeperService<>(\\w+)" + name: "zookeeper_$2" diff --git a/service.json b/service.json new file mode 100644 index 0000000..ad4b762 --- /dev/null +++ b/service.json @@ -0,0 +1,131 @@ +{ + "name": "zookeeper", + "jsonversion": "3", + "servicetype": "core", + "description": "zookeeper", + "homepage": "https://github.com/germanedge/zookeeper-docker", + "keywords": [ + "zookeeper", + "kafka" + ], + "author": "Christoph Heuwieser ", + "contributors": [ + "Niels Oldenburg " + ], + "repository": { + "type": "docker", + "url": "germanedge-docker.artifactory.new-solutions.com/edge-one/" + }, + "entrypoint": "entrypoint.sh", + "dockerfile": "Dockerfile", + "version": "3.87.0", + "profiles": { + "production": { + "configuration": { + "ports": [ + { + "containerPort": 2181 + } + ], + "volumes": [ + { + "name": "zookeeper_volume", + "value": "/opt/zookeeper/data", + "size": "4Gi" + } + ], + "environments": [ + { + "name": "EXT_CA_CERT", + "type": "secretRef", + "secretName": "external-ca-cert-secret", + "secretKey": "ext_ca_cert", + "optional": true + }, + { + "name": "K8S_NAMESPACE", + "type": "fieldRef", + "fieldPath": "metadata.namespace" + } + ], + "command": [], + "depends_on": [ + "consul" + ], + "deploy": { + "mode": { + "value": "replicated" + }, + "replicas": { + "value": 1 + }, + "resources": { + "limits": { + "memory": "512M" + } + }, + "labels": [], + "placement": { + "constraints": [ + "node.platform.os == linux" + ] + }, + "restart_policy": { + "condition": "on-failure", + "delay": "30s" + } + } + }, + "dependencies": { + "consul": ">=1.7.1 <2.0.0" + } + }, + "development": { + "configuration": { + "ports": [ + { + "containerPort": 2181 + } + ], + "volumes": [ + { + "name": "zookeeper_volume", + "value": "/opt/zookeeper/data", + "size": "4Gi" + } + ], + "environments": [], + "command": [], + "depends_on": [ + "consul" + ], + "deploy": { + "mode": { + "value": "replicated" + }, + "replicas": { + "value": 1 + }, + "resources": { + "limits": { + "memory": "512M" + } + }, + "labels": [], + "placement": { + "constraints": [ + "node.platform.os == linux" + ] + }, + "restart_policy": { + "condition": "on-failure", + "delay": "30s" + } + } + }, + "dependencies": { + "consul": ">=1.7.1 <2.0.0" + } + } + } +} diff --git a/start-zk.sh b/start-zk.sh index df6ad64..bbfbef2 100755 --- a/start-zk.sh +++ b/start-zk.sh @@ -1,4 +1,12 @@ +#!/bin/sh + sed -i -r 's|#(log4j.appender.ROLLINGFILE.MaxBackupIndex.*)|\1|g' $ZK_HOME/conf/log4j.properties sed -i -r 's|#autopurge|autopurge|g' $ZK_HOME/conf/zoo.cfg +sed -i -r 's|(zookeeper.root.logger=.*)|\1, ROLLINGFILE|g' $ZK_HOME/conf/log4j.properties +sed -i -r 's|(zookeeper.log.maxfilesize=.*)|zookeeper.log.maxfilesize=16MB|g' $ZK_HOME/conf/log4j.properties +sed -i -r 's|#metricsProvider|metricsProvider|g' $ZK_HOME/conf/zoo.cfg + +export ZOO_LOG4J_PROP="INFO,CONSOLE,ROLLINGFILE" + -/opt/zookeeper-3.4.13/bin/zkServer.sh start-foreground +/opt/zookeeper/bin/zkServer.sh start-foreground diff --git a/startup.sh b/startup.sh new file mode 100755 index 0000000..b3b1707 --- /dev/null +++ b/startup.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +chown root /edgeone/filebeat/filebeat.yml +/usr/bin/start-zk.sh \ No newline at end of file