Merge pull request #163 from christopher-russell/master

Migrate to Helm v3, Change dependencies from deprecated official helm charts to Bitnami charts

Author: arm4b

.circleci/config.yml

@@ -4,10 +4,8 @@ version: 2.1
 orbs:
   # https://circleci.com/orbs/registry/orb/circleci/kubernetes
   kubernetes: circleci/[email protected]
-  # Pins Helm to v2.x
-  # TODO: Consider upgrading Helm to v3.0 (https://github.com/StackStorm/stackstorm-ha/issues/98)
   # https://circleci.com/orbs/registry/orb/circleci/helm
-  helm: circleci/helm@0.2.3
+  helm: circleci/helm@1.1.1
   # https://circleci.com/orbs/registry/orb/ccpgames/minikube
   minikube: ccpgames/[email protected]
 
@@ -16,15 +14,13 @@ jobs:
   helm-lint:
     working_directory: ~/stackstorm-ha
     docker:
-      # Pin Helm to v2.x, see https://github.com/StackStorm/stackstorm-ha/issues/98
-      - image: lachlanevenson/k8s-helm:v2.16.9
+      - image: lachlanevenson/k8s-helm:v3.3.4
     steps:
       - checkout
       - run:
           name: Prepare Helm
           command: |
             set -x
-            helm init --stable-repo-url https://charts.helm.sh/stable --client-only
             helm dependency update
       - run:
           name: Helm Lint Check
@@ -65,33 +61,33 @@ jobs:
       - kubernetes/install
       - minikube/minikube-install:
           # https://github.com/kubernetes/minikube/releases
-          version: v1.10.1
+          version: v1.15.1
       - run:
           name: Install dependencies
           command: |
             sudo apt update || true
             # K8s 1.18 requires conntrack
             # See: https://github.com/kubernetes/minikube/issues/7179
             sudo apt install -y conntrack
+      - run:
+          name: Install Helm v3
+          command: curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
       - run:
           name: Create new K8s cluster
           command: sudo -E minikube start --vm-driver=none
           environment:
             CHANGE_MINIKUBE_NONE_USER: true
-      - run:
-          name: Helm Init
-          command: |
-            curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash
-            helm init --stable-repo-url https://charts.helm.sh/stable
       - run:
           name: Update stackstorm-ha chart dependencies
           command: helm dependency update
       - run:
           name: Helm install stackstorm-ha chart
-          command: helm install --timeout 600 --debug --wait --name stackstorm-ha .
+          command: helm install --timeout 10m0s --debug --wait --name-template stackstorm-ha .
       - run:
+          # once https://github.com/helm/community/pull/165 is merged we will need to add the parallelism flag back into the
+          # below command
           name: Helm test
-          command: helm test stackstorm-ha --parallel --cleanup
+          command: helm test stackstorm-ha
       - run:
           when: always
           name: Show created K8s resources

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## In Development
 
+## v0.50.0
+* Drop Helm `v2` support and fully migrate to Helm `v3` (#163)
+* Switch dependencies from deprecated `helm/charts` to new Bitnami Subcharts (#163)
+
 ## v0.41.0
 * Fix Helm 2 repository location to a new working URL https://charts.helm.sh/stable (#164) (by @manisha-tanwar)
 

Chart.yaml

@@ -1,8 +1,8 @@
-apiVersion: v1
-# Update StackStorm version here to rely on other Docker images tags
+apiVersion: v2
+# StackStorm version which refers to Docker images tag
 appVersion: 3.4dev
 name: stackstorm-ha
-version: 0.41.0
+version: 0.50.0
 description: StackStorm K8s Helm Chart, optimized for running StackStorm in HA environment.
 home: https://stackstorm.com/
 icon: https://landscape.cncf.io/logos/stack-storm.svg
@@ -27,3 +27,20 @@ details:
   This Helm chart is a fully installable app that codifies StackStorm cluster deployment optimized for HA and K8s environment.
   RabbitMQ-HA, MongoDB-HA clusters and coordination backend st2 relies on will be deployed as 3rd party chart dependencies.
   For configuration details please check default values.yaml and README.
+dependencies:
+  - name: rabbitmq
+    version: 8.0.2
+    repository: https://charts.bitnami.com/bitnami
+    condition: rabbitmq.enabled
+  - name: mongodb
+    version: 10.0.1
+    repository: https://charts.bitnami.com/bitnami
+    condition: mongodb.enabled
+  - name: external-dns
+    version: 4.0.0
+    repository: https://charts.bitnami.com/bitnami
+    condition: external-dns.enabled
+  - name: etcd
+    version: 5.1.0
+    repository: https://charts.bitnami.com/bitnami
+    condition: etcd.enabled

README.md

@@ -10,7 +10,7 @@ It's more than welcome to fine-tune each component settings to fit specific avai
 
 ## Requirements
 * [Kubernetes](https://kubernetes.io/docs/setup/pick-right-solution/) cluster
-* [Helm](https://docs.helm.sh/using_helm/#install-helm) and [Tiller](https://docs.helm.sh/using_helm/#initialize-helm-and-install-tiller) `v2.x`
+* [Helm](https://docs.helm.sh/using_helm/#install-helm) `v3.x`
 
 ## Usage
 1) Edit `values.yaml` with configuration for the StackStorm HA K8s cluster.
@@ -157,10 +157,10 @@ StackStorm ChatOps service, based on hubot engine, custom stackstorm integration
 Due to Hubot limitation, st2chatops doesn't provide mechanisms to guarantee high availability and so only single `1` node of st2chatops is deployed.
 This service is disabled by default. Please refer to Helm `values.yaml` about how to enable and configure st2chatops with ENV vars for your preferred chat service.
 
-### [MongoDB HA ReplicaSet](https://github.com/helm/charts/tree/master/stable/mongodb-replicaset)
-StackStorm works with MongoDB as a database engine. External Helm Chart is used to configure MongoDB HA [ReplicaSet](https://docs.mongodb.com/manual/tutorial/deploy-replica-set/).
+### [MongoDB ReplicaSet](https://github.com/bitnami/charts/tree/master/bitnami/mongodb)
+StackStorm works with MongoDB as a database engine. External Helm Chart is used to configure MongoDB [ReplicaSet](https://docs.mongodb.com/manual/tutorial/deploy-replica-set/).
 By default `3` nodes (1 primary and 2 secondaries) of MongoDB are deployed via K8s StatefulSet.
-For more advanced MongoDB configuration, refer to official [mongodb-replicaset](https://github.com/helm/charts/tree/master/stable/mongodb-replicaset)
+For more advanced MongoDB configuration, refer to bitnami [mongodb](https://github.com/bitnami/charts/tree/master/bitnami/mongodb)
 Helm chart settings, which might be fine-tuned via `values.yaml`.
 
 The deployment of MongoDB to the k8s cluster can be disabled by setting the mongodb-ha.enabled key in values.yaml to false.  *Note: Stackstorm relies heavily on connections to a MongoDB instance.  If the in-cluster deployment of MongoDB is disabled, a connection to an external instance of MongoDB must be configured.  The st2.config key in values.yaml provides a way to configure stackstorm.  See [Configure MongoDB](https://docs.stackstorm.com/install/config/config.html#configure-mongodb) for configuration details.*
@@ -169,14 +169,14 @@ The deployment of MongoDB to the k8s cluster can be disabled by setting the mong
 RabbitMQ is a message bus StackStorm relies on for inter-process communication and load distribution.
 External Helm Chart is used to deploy [RabbitMQ cluster](https://www.rabbitmq.com/clustering.html) in Highly Available mode.
 By default `3` nodes of RabbitMQ are deployed via K8s StatefulSet.
-For more advanced RabbitMQ configuration, please refer to official [rabbitmq-ha](https://github.com/helm/charts/tree/master/stable/rabbitmq-ha)
+For more advanced RabbitMQ configuration, please refer to bitnami [rabbitmq](https://github.com/bitnami/charts/tree/master/bitnami/rabbitmq)
 Helm chart repository, - all settings could be overridden via `values.yaml`.
 
 The deployment of RabbitMQ to the k8s cluster can be disabled by setting the rabbitmq-ha.enabled key in values.yaml to false.  *Note: Stackstorm relies heavily on connections to a RabbitMQ instance.  If the in-cluster deployment of RabbitMQ is disabled, a connection to an external instance of RabbitMQ must be configured.  The st2.config key in values.yaml provides a way to configure stackstorm.  See [Configure RabbitMQ](https://docs.stackstorm.com/install/config/config.html#configure-rabbitmq) for configuration details.*
 
 ### [etcd](https://docs.stackstorm.com/latest/reference/ha.html#zookeeper-redis)
 StackStorm employs etcd as a distributed coordination backend, required for st2 cluster components to work properly in HA scenario.
-`3` node Raft cluster is deployed via external official Helm chart dependency [etcd-operator](https://github.com/helm/charts/tree/master/stable/etcd-operator).
+`3` node Raft cluster is deployed via external bitnami Helm chart dependency [etcd](https://github.com/bitnami/charts/tree/master/bitnami/etcd).
 As any other Helm dependency, it's possible to further configure it for specific scaling needs via `values.yaml`.
 
 ## Install custom st2 packs in the cluster

conf/rabbit-definition.conf

@@ -0,0 +1,35 @@
+{
+  "users": [
+    {
+      "name": {{ .Values.rabbitmq.auth.username | quote }},
+      "password": {{ .Values.rabbitmq.auth.password | quote }},
+      "tags": "administrator"
+    }
+  ],
+  "permissions": [
+    {
+      "user": {{ .Values.rabbitmq.auth.username | quote }},
+      "vhost": {{ .Values.rabbitmq.ingress.path | quote }},
+      "configure": ".*",
+      "write": ".*",
+      "read": ".*"
+    }
+  ],
+  "vhosts": [
+    {
+      "name": "/"
+    }
+  ],
+  "policies": [
+    {
+      "vhost":"/",
+      "name":"ha",
+      "pattern":"",
+      "definition": {
+        "ha-mode":"all",
+        "ha-sync-mode": "automatic",
+        "ha-sync-batch-size":10
+      }
+    }
+  ]
+}

requirements.yaml

@@ -50,7 +50,7 @@ Create the name of the stackstorm-ha service account to use
 # Allow calling helpers from nested sub-chart
 # https://stackoverflow.com/a/52024583/4533625
 # https://github.com/helm/helm/issues/4535#issuecomment-477778391
-# Usage: "{{ include "nested" (list . "mongodb-ha" "mongodb-replicaset.fullname") }}"
+# Usage: "{{ include "nested" (list . "mongodb" "mongodb.fullname") }}"
 {{- define "nested" }}
 {{- $dot := index . 0 }}
 {{- $subchart := index . 1 | splitList "." }}
@@ -63,11 +63,16 @@ Create the name of the stackstorm-ha service account to use
 {{- end }}
 
 # Generate comma-separated list of nodes for MongoDB-HA connection string, based on number of replicas and service name
-{{- define "mongodb-ha-nodes" -}}
-{{- $replicas := (int (index .Values "mongodb-ha" "replicas")) }}
-{{- $mongo_fullname := include "nested" (list $ "mongodb-ha" "mongodb-replicaset.fullname") }}
-  {{- range $index0 := until $replicas -}}
-    {{- $index1 := $index0 | add1 -}}
-{{ $mongo_fullname }}-{{ $index0 }}.{{ $mongo_fullname }}{{ if ne $index1 $replicas }},{{ end }}
+{{- define "mongodb-nodes" -}}
+{{- $replicas := (int (index .Values "mongodb" "replicaCount")) }}
+{{- $architecture := (index .Values "mongodb" "architecture" ) }}
+{{- $mongo_fullname := include "nested" (list $ "mongodb" "mongodb.fullname") }}
+{{- range $index0 := until $replicas -}}
+  {{- $index1 := $index0 | add1 -}}
+  {{- if eq $architecture "replicaset" }}
+    {{- $mongo_fullname }}-{{ $index0 }}.{{ $mongo_fullname }}-headless{{ if ne $index1 $replicas }},{{ end }}
+  {{- else }}
+    {{- $mongo_fullname }}-{{ $index0 }}.{{ $mongo_fullname }}{{ if ne $index1 $replicas }},{{ end }}
   {{- end -}}
 {{- end -}}
+{{- end -}}

templates/_helpers.tpl

@@ -20,24 +20,25 @@ data:
   st2.docker.conf: |
     [auth]
     api_url = http://{{ .Release.Name }}-st2api{{ template "enterpriseSuffix" . }}:9101/
-    {{- if index .Values "etcd-operator" "enabled" }}
+    {{- if index .Values "etcd" "enabled" }}
     [coordination]
-    url = etcd://{{ index .Values "etcd-operator" "etcdCluster" "name" }}-client:2379
+    url = etcd://{{ index .Values "etcd" "fullnameOverride" }}:2379
     {{- end }}
-    {{- if index .Values "rabbitmq-ha" "enabled" }}
+    {{- if index .Values "rabbitmq" "enabled" }}
     [messaging]
-    url = amqp://{{ required "rabbitmq-ha.rabbitmqUsername is required!" (index .Values "rabbitmq-ha" "rabbitmqUsername") }}:{{ required "rabbitmq-ha.rabbitmqPassword is required!" (index .Values "rabbitmq-ha" "rabbitmqPassword") }}@{{ .Release.Name }}-rabbitmq-ha-discovery:5672
+    url = amqp://{{ required "rabbitmq.auth.username is required!" (index .Values "rabbitmq" "auth" "username") }}:{{ required "rabbitmq.auth.password is required!" (index .Values "rabbitmq" "auth" "password") }}@{{ .Release.Name }}-rabbitmq:5672{{ required "rabbitmq.ingress.path is required!" (index .Values "rabbitmq" "ingress" "path") }}
     {{- end }}
-    {{- if index .Values "mongodb-ha" "enabled" }}
+    {{- if index .Values "mongodb" "enabled" }}
     [database]
-    {{- if index .Values "mongodb-ha" "auth" "enabled" }}
-    host = mongodb://{{ template "mongodb-ha-nodes" $ }}/?authSource=admin&replicaSet={{ index .Values "mongodb-ha" "replicaSetName" }}
-    username = {{ required "mongodb-ha.auth.adminUser is required!" (index .Values "mongodb-ha" "auth" "adminUser") }}
-    password = {{ required "mongodb-ha.auth.adminPassword is required!" (index .Values "mongodb-ha" "auth" "adminPassword") }}
+    {{- if index .Values "mongodb" "auth" "enabled" }}
+    host = mongodb://{{ template "mongodb-nodes" $ }}/{{ required "mongodb.auth.database is required!" (index .Values "mongodb" "auth" "database") }}?authSource={{ required "mongodb.auth.database is required!" (index .Values "mongodb" "auth" "database") }}&replicaSet={{ index .Values "mongodb" "replicaSetName" }}
+    username = {{ required "mongodb.auth.username is required!" (index .Values "mongodb" "auth" "username") }}
+    password = {{ required "mongodb.auth.password is required!" (index .Values "mongodb" "auth" "password") }}
+    db_name = {{ required "mongodb.auth.database is required!" (index .Values "mongodb" "auth" "database") }}
     {{- else }}
-    host = mongodb://{{ template "mongodb-ha-nodes" $ }}/?replicaSet={{ index .Values "mongodb-ha" "replicaSetName" }}
+    host = mongodb://{{ template "mongodb-nodes" $ }}/?replicaSet={{ index .Values "mongodb" "replicaSetName" }}
     {{- end }}
-    port = {{ index .Values "mongodb-ha" "port" }}
+    port = {{ index .Values "mongodb" "service" "port" }}
     {{- end }}
     {{- if .Values.secrets.st2.datastore_crypto_key }}
     [keyvalue]

templates/configmaps_st2-conf.yaml

@@ -0,0 +1,21 @@
+# This configuration is a workaround to https://github.com/bitnami/charts/issues/4635
+# This code block should be dropped once the above issue is resolved and definitions can be defined as shown in 
+# https://github.com/bitnami/charts/tree/master/bitnami/rabbitmq#load-definitions
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Release.Name }}-rabbitmq-definitions
+  annotations:
+    description: A rabbitmq definition which will be loaded by the rabbitmq subchart to enable mirroring for Rabbit HA
+  labels:
+    app: st2
+    tier: backend
+    vendor: stackstorm
+    support: {{ template "supportMethod" . }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+type: Opaque
+data:
+  rabbitmq-definitions.json:  {{ tpl (.Files.Get "conf/rabbit-definition.conf") . | b64enc }}

templates/secrets_rabbitmq.yaml

@@ -12,6 +12,7 @@ metadata:
     heritage: {{ .Release.Service }}
   annotations:
     "helm.sh/hook": test-success
+    "helm.sh/hook-delete-policy": hook-succeeded
 spec:
   {{- if .Values.enterprise.enabled }}
   imagePullSecrets: