[bitnami/redis-cluster]: add preStop hook that gracefully fails over master nodes on pod termination

Author: sobotklp

bitnami/redis-cluster/Chart.yaml

@@ -35,4 +35,4 @@ maintainers:
 name: redis-cluster
 sources:
 - https://github.com/bitnami/charts/tree/main/bitnami/redis-cluster
-version: 13.0.4
+version: 13.1.0

bitnami/redis-cluster/README.md

@@ -621,6 +621,7 @@ See [#15075](https://github.com/bitnami/charts/issues/15075)
 | `cluster.init`                                            | Enable the initialization of the Redis(R) Cluster                                                          | `true`         |
 | `cluster.nodes`                                           | The number of master nodes should always be >= 3, otherwise cluster creation will fail                     | `6`            |
 | `cluster.replicas`                                        | Number of replicas for every master in the cluster                                                         | `1`            |
+| `cluster.redisShutdownWaitFailover`                       | Whether the Redis(R) master container waits for the failover at shutdown.                                  | `true`         |
 | `cluster.externalAccess.enabled`                          | Enable access to the Redis                                                                                 | `false`        |
 | `cluster.externalAccess.hostMode`                         | Set cluster preferred endpoint type as hostname                                                            | `false`        |
 | `cluster.externalAccess.service.disableLoadBalancerIP`    | Disable use of `Service.spec.loadBalancerIP`                                                               | `false`        |

bitnami/redis-cluster/templates/redis-statefulset.yaml

@@ -268,6 +268,14 @@ spec:
           {{- end }}
           {{- if .Values.redis.lifecycleHooks }}
           lifecycle: {{- include "common.tplvalues.render" (dict "value" .Values.redis.lifecycleHooks "context" $) | nindent 12 }}
+          {{- else }}
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                  - /bin/bash
+                  - -ec
+                  - /opt/bitnami/scripts/start-scripts/prestop-redis-cluster.sh
           {{- end }}
           {{- end }}
           {{- if .Values.redis.resources }}

bitnami/redis-cluster/templates/scripts-configmap.yaml

@@ -105,3 +105,66 @@ data:
       echo "$response"
       exit 1
     fi
+  prestop-redis-cluster.sh: |-
+    #!/bin/bash
+    set -e
+
+    # redis-cli automatically consumes credentials from the REDISCLI_AUTH variable
+    [[ -n "$REDIS_PASSWORD" ]] && export REDISCLI_AUTH="$REDIS_PASSWORD"
+    [[ -f "$REDIS_PASSWORD_FILE" ]] && export REDISCLI_AUTH="$(< "${REDIS_PASSWORD_FILE}")"
+
+    run_redis_command() {
+        local args=("-h" "$1")
+        shift
+    {{- if .Values.tls.enabled }}
+        args+=("--tls" "--cert" "{{ template "redis-cluster.tlsCert" . }}" "--key" "{{ template "redis-cluster.tlsCertKey" . }}" "--cacert" "{{ template "redis-cluster.tlsCACert" . }}" "-p" $REDIS_TLS_PORT_NUMBER)
+    {{- else }}
+        args+=("-p" $REDIS_PORT_NUMBER)
+    {{- end }}
+        redis-cli "${args[@]}" "$@"
+    }
+
+    is_master() {
+        REDIS_ROLE=$(run_redis_command 127.0.0.1 ROLE | head -1)
+        [[ "$REDIS_ROLE" == "master" ]]
+    }
+
+    get_cluster_node_id() {
+        CLUSTER_ID=$(run_redis_command 127.0.0.1 CLUSTER MYID | head -1)
+        echo $CLUSTER_ID
+    }
+
+    get_replica_ips() {
+      CLUSTER_NODE_ID=$(get_cluster_node_id)
+
+      # Get a list of replica IP addresses that could be promoted
+      # Use shuf to randomize the order
+      REPLICAS="$(run_redis_command 127.0.0.1 CLUSTER REPLICAS $CLUSTER_NODE_ID | grep -Ev "(disconnected|nofailover|noaddr|handshake)" | cut -d " " -f 2 | cut -d ":" -f 1 | shuf)"
+      echo "$REPLICAS"
+    }
+
+    if is_master; then
+      # Get list of replicas of the current master
+      mapfile -t REPLICA_IPS < <( get_replica_ips )
+
+      NUM_REPLICAS=${#REPLICA_IPS[@]}
+      echo "Found $NUM_REPLICAS available replicas"
+
+      if (( $NUM_REPLICAS > 0 )); then
+        # Iterate over replicas, attempting to promote one
+        for REPLICA_IP in "${REPLICA_IPS[@]}"; do
+          echo "Going to fail over to replica at $REPLICA_IP"
+          result=$(run_redis_command $REPLICA_IP CLUSTER FAILOVER)
+
+          if [[ "$result" == "OK" ]]; then
+            {{- if .Values.cluster.redisShutdownWaitFailover }}
+            # Wait for clients to update their topology
+            sleep 10
+            break
+            {{- end }}
+          fi
+        done
+      fi
+    else
+      exit 0
+    fi

bitnami/redis-cluster/values.yaml

@@ -873,6 +873,10 @@ cluster:
   replicas: 1
   ## Configuration to access the Redis(R) Cluster from outside the Kubernetes cluster
   ##
+  ## @param cluster.redisShutdownWaitFailover Whether the Redis(R) master container waits for the failover at shutdown.
+  ##
+  redisShutdownWaitFailover: true
+
   externalAccess:
     ## @param cluster.externalAccess.enabled Enable access to the Redis
     ##