Add TCP health checks and idempotent self-registration for Skip services

Author: bennostein

examples/hackernews/kubernetes/skip.yaml

@@ -48,16 +48,23 @@ spec:
 
           if [[ $id -eq 0 ]]; then
             export SKIP_LEADER=true
-            echo "set server skip_control/leader addr $ip:8081 ; enable server skip_control/leader" | socat stdio tcp4-connect:haproxy-kubernetes-ingress:9999
+            echo "set server skip_control/leader addr $ip port 8081 ; enable server skip_control/leader " | socat stdio tcp4-connect:haproxy-kubernetes-ingress:9999
           else
             export SKIP_FOLLOWER=true
             export SKIP_RESOURCE_PREFIX=follower$id
             export SKIP_LEADER_HOST=rhn-skip-0.rhn-skip.default.svc.cluster.local
+            # Self-register both the control and event streaming server with the haproxy load balancer.
+            # Calling 'set server' after 'add server' is redundant on initial scale-up, but necessary for subsequent scale-ups when a server of that name already exists.
+            # Enabling HAProxy TCP health checks ensures that servers are taken out of rotation when the system scales down or instances crash/disconnect for other reasons.
             echo "\
-            add server skip_control/follower$id $ip:8081 ;\
+            add server skip_control/follower$id $ip:8081 check ;\
+            set server skip_control/follower$id addr $ip port 8081 ;\
             enable server skip_control/follower$id ;\
-            add server skip_stream/follower$id $ip:8080 ;\
-            enable server skip_stream/follower$id\
+            enable health skip_control/follower$id ;\
+            add server skip_stream/follower$id $ip:8080 check ;\
+            set server skip_stream/follower$id addr $ip port 8080 ;\
+            enable server skip_stream/follower$id ;\
+            enable health skip_stream/follower$id\
             " | socat stdio tcp4-connect:haproxy-kubernetes-ingress:9999
           fi
           npm start
@@ -77,5 +84,5 @@ spec:
             - wget
             - "--spider"
             - http://localhost:8081/v1/healthcheck
-          initialDelaySeconds: 5
-          periodSeconds: 5
+          initialDelaySeconds: 1
+          periodSeconds: 2