Merge pull request #78 from Yolean/outside-services

Outside (i.e. cluster-external) access through per-broker Service

Author: solsson

10broker-config.yml

@@ -22,6 +22,16 @@ data:
       else
         sed -i "s/#init#broker.rack=#init#/broker.rack=$ZONE/" /etc/kafka/server.properties
       fi
+
+      kubectl -n $POD_NAMESPACE label pod $POD_NAME kafka-broker-id=$KAFKA_BROKER_ID
+
+      OUTSIDE_HOST=$(kubectl get node "$NODE_NAME" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}')
+      if [ $? -ne 0 ]; then
+        echo "Outside (i.e. cluster-external access) host lookup command failed"
+      else
+        OUTSIDE_HOST=${OUTSIDE_HOST}:3240${KAFKA_BROKER_ID}
+        sed -i "s|#init#advertised.listeners=OUTSIDE://#init#|advertised.listeners=OUTSIDE://${OUTSIDE_HOST}|" /etc/kafka/server.properties
+      fi
     }
 
   server.properties: |-
@@ -58,14 +68,18 @@ data:
     #   EXAMPLE:
     #     listeners = PLAINTEXT://your.host.name:9092
     #listeners=PLAINTEXT://:9092
+    listeners=OUTSIDE://:9094,PLAINTEXT://:9092
 
     # Hostname and port the broker will advertise to producers and consumers. If not set,
     # it uses the value for "listeners" if configured.  Otherwise, it will use the value
     # returned from java.net.InetAddress.getCanonicalHostName().
     #advertised.listeners=PLAINTEXT://your.host.name:9092
+    #init#advertised.listeners=OUTSIDE://#init#,PLAINTEXT://:9092
 
     # Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details
     #listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL
+    listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL,OUTSIDE:PLAINTEXT
+    inter.broker.listener.name=PLAINTEXT
 
     # The number of threads that the server uses for receiving requests from the network and sending responses to the network
     num.network.threads=3

50kafka.yml

@@ -21,6 +21,14 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
         command: ['/bin/bash', '/etc/kafka/init.sh']
         volumeMounts:
         - name: config
@@ -32,7 +40,10 @@ spec:
         - name: KAFKA_LOG4J_OPTS
           value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties
         ports:
-        - containerPort: 9092
+        - name: inside
+          containerPort: 9092
+        - name: outside
+          containerPort: 9094
         command:
         - ./bin/kafka-server-start.sh
         - /etc/kafka/server.properties

outside-services/outside-0.yml

@@ -0,0 +1,15 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: outside-0
+  namespace: kafka
+spec:
+  selector:
+    app: kafka
+    kafka-broker-id: "0"
+  ports:
+  - protocol: TCP
+    targetPort: 9094
+    port: 32400
+    nodePort: 32400
+  type: NodePort

outside-services/outside-1.yml

@@ -0,0 +1,15 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: outside-1
+  namespace: kafka
+spec:
+  selector:
+    app: kafka
+    kafka-broker-id: "1"
+  ports:
+  - protocol: TCP
+    targetPort: 9094
+    port: 32401
+    nodePort: 32401
+  type: NodePort

outside-services/outside-2.yml

@@ -0,0 +1,15 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: outside-2
+  namespace: kafka
+spec:
+  selector:
+    app: kafka
+    kafka-broker-id: "2"
+  ports:
+  - protocol: TCP
+    targetPort: 9094
+    port: 32402
+    nodePort: 32402
+  type: NodePort