mongo-set.yaml

#------------------------------------------------------------------
#       Copyright 2020, noelmcloughlin
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

###------------------------------------------------------------###
### Deploys a mongoDB replicaset, five replicas, no sidecar.   ###
### Uses official mongo docker image (without the entrypoint). ###
###                                                            ###
### Author: noelmcloughlin                                     ###
### If you wait to do everything until you're sure it's right, ###
### you'll probably never do much of anything” ~ Win Borden    ###
###                                                            ###
###------------------------------------------------------------###

###  Testing:

# $ MONG=$(kubectl cluster-info |awk -F: '{print $2}' |sed -e s@/@@g |head -1)
# $ mongo --host $MONG:27017 -u myroot -p myroot

###  Refs:
#   https://docs.mongodb.com/manual/replication/#replication-in-mongodb
#   https://medium.com/codeops/how-to-setup-a-mongodb-replica-set-918f21da50ed
#   http://kubernetes.kansea.com/docs/user-guide/petset/bootstrapping
#   https://kubernetes.io/docs/concepts/cluster-administration/logging
#   https://doc.akka.io/docs/akka-management/current/bootstrap/kubernetes.html
#   https://www.bogotobogo.com/DevOps/Docker/Docker_MongoDB_with_StatefulSets_on_GCP_Kubernetes.php
#   https://github.com/lijiaocn/k8s-mongo-petset/blob/master/mongo-init/entrypoint.sh
#   https://kubernetes.io/docs/concepts/services-networking/dns-pod-service

# STORAGE CLASS
# ... delay volume binding until pods are scheduled

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: local
  labels:
    type: local
  annotations:
    storageclass.beta.kubernetes.io/is-default-class: "true"
# provisioner: kubernetes.io/no-provisioner
provisioner: k8s.io/minikube-hostpath
parameters:
  pool: default
  description: Kubernetes volume
  fsType: xfs
  adminSecretNamespace: default
  adminSecretName: localos-secret
reclaimPolicy: Delete
volumeBindingMode: WaitForFirstConsumer
allowVolumeExpansion: false
---
# PERSISTENT VOLUMES

apiVersion: v1
kind: PersistentVolume
metadata:
  name: mongo-pv0
  labels:
    app: mongo
    bind: mongo
  annotations:
    "volume.alpha.kubernetes.io/node-affinity": '{
      "requiredDuringSchedulingIgnoredDuringExecution": {
        "nodeSelectorTerms": [
          { "matchExpressions": [
              { "key": "kubernetes.io/hostname",
                "operator": "In",
                "values": ["minikube"]
              }
          ]}
        ]}}'
spec:
  capacity:
    storage: 500Mi
  accessModes:
    - ReadWriteOnce
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Retain
  storageClassName: local
  hostPath:
    type: DirectoryOrCreate
    path: /tmp/mongo-pv0
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - minikube
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mongo-pv1
  labels:
    app: mongo
    bind: mongo
  annotations:
    "volume.alpha.kubernetes.io/node-affinity": '{
      "requiredDuringSchedulingIgnoredDuringExecution": {
        "nodeSelectorTerms": [
          { "matchExpressions": [
              { "key": "kubernetes.io/hostname",
                "operator": "In",
                "values": ["minikube"]
              }
          ]}
        ]}}'
spec:
  capacity:
    storage: 500Mi
  accessModes:
    - ReadWriteOnce
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Retain
  storageClassName: local
  hostPath:
    type: DirectoryOrCreate
    path: /tmp/mongo-pv1
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - minikube
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mongo-pv2
  labels:
    app: mongo
    bind: mongo
  annotations:
    "volume.alpha.kubernetes.io/node-affinity": '{
      "requiredDuringSchedulingIgnoredDuringExecution": {
        "nodeSelectorTerms": [
          { "matchExpressions": [
              { "key": "kubernetes.io/hostname",
                "operator": "In",
                "values": ["minikube"]
              }
          ]}
        ]}}'
spec:
  capacity:
    storage: 500Mi
  accessModes:
    - ReadWriteOnce
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Retain
  storageClassName: local
  hostPath:
    type: DirectoryOrCreate
    path: /tmp/mongo-pv2
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - minikube
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mongo-pv3
  labels:
    app: mongo
    bind: mongo
  annotations:
    "volume.alpha.kubernetes.io/node-affinity": '{
      "requiredDuringSchedulingIgnoredDuringExecution": {
        "nodeSelectorTerms": [
          { "matchExpressions": [
              { "key": "kubernetes.io/hostname",
                "operator": "In",
                "values": ["minikube"]
              }
          ]}
        ]}}'
spec:
  capacity:
    storage: 500Mi
  accessModes:
    - ReadWriteOnce
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Retain
  storageClassName: local
  hostPath:
    type: DirectoryOrCreate
    path: /tmp/mongo-pv3
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - minikube
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mongo-pv4
  labels:
    app: mongo
    bind: mongo
  annotations:
    "volume.alpha.kubernetes.io/node-affinity": '{
      "requiredDuringSchedulingIgnoredDuringExecution": {
        "nodeSelectorTerms": [
          { "matchExpressions": [
              { "key": "kubernetes.io/hostname",
                "operator": "In",
                "values": ["minikube"]
              }
          ]}
        ]}}'
spec:
  capacity:
    storage: 500Mi
  accessModes:
    - ReadWriteOnce
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Retain
  storageClassName: local
  hostPath:
    type: DirectoryOrCreate
    path: /tmp/mongo-pv4
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - minikube
---
# SERVICE
## Headless service for stable dns names
##   mongo-0.mongo, mongo-1.mongo, mongo-2.mongo

apiVersion: v1
kind: Service
metadata:
  name: mongo
  labels:
    name: mongo
spec:
  ports:
    - name: mongo-default
      port: 27017
      protocol: TCP
      targetPort: 27017
  selector:
    role: mongo
  clusterIP: None
---
apiVersion: v1
kind: Service
metadata:
  name: mongodb
  labels:
    name: mongodb
spec:
  type: NodePort
  ports:
    - name: mongo-default
      protocol: TCP
      port: 27017
      nodePort: 27017
      targetPort: 27017
  selector:
    role: mongo
---
# CONFIGMAPS

apiVersion: v1
kind: ConfigMap
metadata:
  name: initcontainer
  namespace: default
data:
  utils.js: |+
    function env(myenv, def) {
       var ret= run("sh", "-c", `printenv -0 ${myenv} >/tmp/${myenv}.txt`);
       if (ret != 0) return def;
       return cat(`/tmp/${myenv}.txt`)
    }

  primary.js: |-
    config = {
        _id : "rs0",
        members : [
             {_id : 0, host : 'mongo-0.mongo.default.svc.cluster.local:27017', priority: 1000}
        ] };
    rs.initiate(config);
    sleep(1)

  members.js: |-
    sleep(1)
    rs.add({_id : 1, host : 'mongo-1.mongo.default.svc.cluster.local:27017', priority: 1})
    rs.add({_id : 2, host : 'mongo-2.mongo.default.svc.cluster.local:27017', priority: 2})
    rs.add({_id : 3, host : 'mongo-3.mongo.default.svc.cluster.local:27017', priority: 3})
    rs.add({_id : 4, host : 'mongo-4.mongo.default.svc.cluster.local:27017', priority: 4})
    sleep(1)

  secondary.js: |-
    rs.slaveOk()

  users.js: |+
    load('/mnt/state/init/utils.js');
    db.createUser(
      {
        user: env('MONGO_INITDB_ROOT_USERNAME'),
        pwd: env('MONGO_INITDB_ROOT_PASSWORD'),
        roles: [
            { role: "root", db: env('MONGO_INITDB_DATABASE')},
            { role: "clusterAdmin", db: env('MONGO_INITDB_DATABASE')},
            { role: "__system", db: env('MONGO_INITDB_DATABASE')}
        ] });
    db.createUser(
      {
        user: env('MONGO_INITDB_DEF_USERNAME'),
        pwd: env('MONGO_INITDB_DEF_PASSWORD'),
        roles: [
            { role: "root", db: env('MONGO_INITDB_DATABASE')},
            { role: "clusterAdmin", db: env('MONGO_INITDB_DATABASE')}
        ] });
    sleep(1)

  mongod.conf: |-
    security:
       # keyFile: /mnt/state/keyfile
       authorization: disabled
    storage:
       dbPath: /var/lib/mongodb

  runinit.sh: |+
    set -xe
    date >> /var/lib/mongodb/init.log
    [ "${DELETE_EXISTING_DATA}" == "yes" ] && rm -rf /var/lib/mongodb/*
    cp -p /mnt/initcontainer/* /mnt/state/init/
    cp -p /mnt/runcontainer/* /mnt/state/
    chmod 400 /mnt/state/keyfile || true
    chown -R mongodb /mnt/state

    mongod --config /mnt/state/init/mongod.conf &
    sleep 1 # keepme or crash
    mongo localhost:27017/${MONGO_INITDB_DATABASE} /mnt/state/init/users.js
    u=${MONGO_INITDB_DEF_USERNAME} && p=${MONGO_INITDB_DEF_PASSWORD}
    sleep 1 # keepme or crash
    if [ "${HOSTNAME}" = "mongo-0" ]; then
        mongo localhost:27017/${MONGO_INITDB_DATABASE} -u ${u} -p ${p} /mnt/state/init/primary.js
    fi
    sync && sleep 1 && sync

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: runcontainer
  namespace: default
data:
  keyfile: |-
    Te/urcU4Pww+duwgVsrntSUkZrIMCmak8LTDtm0L8SzLBQGeHP01RdQ/uhnwAdIb
    8u8bg7k3t3ghZqvoZIfjDLlfFhQUf4QxUz0aJSiEEcAbo9ExArVbQzk8J7+yIRtu
    A6DzBNaj5Bzd69OwcDMXEONc29s+SCsEO2SVuvDtDwF2K3UAqerEv1qlVNzURPbs
    e60WU1/qpmX9urSRNNmdYA6neUQceSR0lJclDUzwzURRIMR5Xk534JpAbRcQ3DkJ
    KraQKusoLNmej6WIM8y2qcviwsEdpvBDJN0c4eZdyatlHkAwYrGTwBnKTeD7J+ik
    ZTehYF7+okMeX4xfVsSV2rYJzSU4XLugEDkKQzxUdHn1eMGQguNzeAHwyeloUoJc
    4GbwWFzTkBJZ4obhtIzcbHezDS1Aq32SbSuqvSPTSq5gGtwVeOtDW8N7YFy5UVdB
    YiLcZm/W4kxOu0i+sOwn/xt5jlVslzuQ65bc9oFTHw4K85nHupBUbxWvhWB8HDok
    hxawwHuD3a2ckXZ9nNcjOKM13LTnuZMmzz/Iuxz4/HxF5wajSnRo5yW+fzYO3Una
    /9gqhfxnp220Cxi+u3ArHgJQQfIUvRLwaDV100KfmjYS43HnkfYzqKyRXoLCeQ3O
    Ex/lFUKaVfVOBYDtHxnnOmA/8c/Nd2rO5axY/7CXgMIAEkyLYPaeVSJU2rGCKWwM
    jTujI2rV2dfIUevwzpQoo6+ik8B64bIA2sqfAa8qCnsmezcwKj6l4Vjjb92wkFoF
    iq4u0AMzZCO0vzu/x2Lq4nlu3CgAbLud5Yx3eGl7R1SQQ4oLCVdBvs/hiQKfjc7y
    XqoAFVsTfcoKU3ayTNeF8OpiT97w4uNTbk06B9qn+5E1T9xVq+d1TU7eGjVGszQK
    zHZW/xUXkPL0AVqg9AHNnbMwrGNFsVM5RelC3iqntsO52OEr3jsdBrpHvA98WoRE
    x7KsV/EJMrBHZGMkwc37VBGdgov6r4pfLfOIOOPmaGp8LewK

  mongod.conf: |-
    # mongod.conf
    # for documentation of all options, see:
    #   http://docs.mongodb.org/manual/reference/configuration-options/
    # Where and how to store data.
    storage:
       dbPath: /var/lib/mongodb
       journal:
        enabled: true
    # network interfaces
    net:
       bindIpAll: true
       ipv6: true
       port: 27017
       unixDomainSocket:
         enabled: true
    # https://docs.mongodb.com/manual/core/authentication
    security:
       keyFile: /etc/mongo/keyfile
       authorization: enabled
    replication:
       replSetName: rs0
    setParameter:
       diagnosticDataCollectionEnabled: false

  run.sh: |+
    set -xe
    mongod --config /etc/mongo/mongod.conf &
    u=${MONGO_INITDB_DEF_USERNAME} && p=${MONGO_INITDB_DEF_PASSWORD}
    sleep 5 # keepme or crash!
    if [ "${HOSTNAME}" = "mongo-0" ]; then
        mongo localhost:27017/${MONGO_INITDB_DATABASE} -u ${u} -p ${p} /etc/mongo/init/primary.js
        sleep 5 # keepme or crash!
        mongo localhost:27017/${MONGO_INITDB_DATABASE} -u ${u} -p ${p} /etc/mongo/init/members.js
    else
        mongo localhost:27017/${MONGO_INITDB_DATABASE} -u ${u} -p ${p} /etc/mongo/init/secondary.js
    fi
    wait

  livenessProbe.sh: |+
    echo "mongo --eval db.adminCommand('ping')"    # todo: determine best check

  readinessProbe.sh: |+
    echo "mongo --eval db.adminCommand('ping')"    # todo: determine best check

---
#SECRETS

apiVersion: v1
kind: Secret
metadata:
  name: mongo
type: Opaque
data:
  MONGO_INITDB_ROOT_USERNAME: bXlzeXN0ZW0=
  MONGO_INITDB_ROOT_PASSWORD: bXlzeXN0ZW0=
  MONGO_INITDB_DEF_USERNAME: bXlyb290
  MONGO_INITDB_DEF_PASSWORD: bXlyb290
stringData:
  MONGO_INITDB_ROOT_USERNAME: mysystem
  MONGO_INITDB_ROOT_PASSWORD: mysystem
  MONGO_INITDB_DEF_USERNAME: myroot
  MONGO_INITDB_DEF_PASSWORD: myroot
  MONGO_INITDB_DATABASE: admin

---
# STATEFULSET

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: mongo
spec:
  serviceName: mongo
  replicas: 3
  podManagementPolicy: Parallel  # OrderedReady
  selector:
    matchLabels:
      role: mongo
  template:
    metadata:
      labels:
        role: mongo
    spec:
      terminationGracePeriodSeconds: 10
      shareProcessNamespace: true
      initContainers:
      - name: init
        image: mongo
        imagePullPolicy: IfNotPresent
        command:
        - bash
        - /mnt/initcontainer/runinit.sh
        volumeMounts:
        - name: initstate
          mountPath: /mnt/state/init/
          readOnly: false
        - name: runstate
          mountPath: /mnt/state/
          readOnly: false
        - name: initcontainer
          mountPath: /mnt/initcontainer/
          readOnly: false
        - name: runcontainer
          mountPath: /mnt/runcontainer/
          readOnly: false
        - name: dbdata
          mountPath: /var/lib/mongodb
          readOnly: false
        - name: podinfo
          mountPath: /etc/podinfo/
          readOnly: false
        env:
          - name: DELETE_EXISTING_DATA
            value: "yes"
          - name: ALLOW_EMPTY_PASSWORD
            value: "no"
          - name: MONGO_INITDB_DATABASE
            valueFrom:
              secretKeyRef:
                name: mongo
                key: MONGO_INITDB_DATABASE
          - name: MONGO_INITDB_ROOT_USERNAME
            valueFrom:
              secretKeyRef:
                name: mongo
                key: MONGO_INITDB_ROOT_USERNAME
          - name: MONGO_INITDB_ROOT_PASSWORD
            valueFrom:
              secretKeyRef:
                name: mongo
                key: MONGO_INITDB_ROOT_PASSWORD
          - name: MONGO_INITDB_DEF_USERNAME
            valueFrom:
              secretKeyRef:
                name: mongo
                key: MONGO_INITDB_DEF_USERNAME
          - name: MONGO_INITDB_DEF_PASSWORD
            valueFrom:
              secretKeyRef:
                name: mongo
                key: MONGO_INITDB_DEF_PASSWORD

      containers:
      - name: mongo
        image: mongo
        imagePullPolicy: IfNotPresent
        command:
        - bash
        - /etc/mongo/run.sh
        env:
          - name: DELETE_EXISTING_DATA
            value: "no"
          - name: MONGO_INITDB_DATABASE
            valueFrom:
              secretKeyRef:
                name: mongo
                key: MONGO_INITDB_DATABASE
          - name: MONGO_INITDB_DEF_USERNAME
            valueFrom:
              secretKeyRef:
                name: mongo
                key: MONGO_INITDB_DEF_USERNAME
          - name: MONGO_INITDB_DEF_PASSWORD
            valueFrom:
              secretKeyRef:
                name: mongo
                key: MONGO_INITDB_DEF_PASSWORD
        ports:
          - name: mongo-default
            containerPort: 27017
        resources:
          limits:
            memory: 500Mi
          requests:
            memory: 500Mi
        lifecycle:
          preStop:
            exec:
              command:
              - /bin/sh
              - '-c'
              - echo 'goodbye world'
        livenessProbe:
          exec:
            command:
            - bash
            - /etc/mongo/livenessProbe.sh
          initialDelaySeconds: 60
          periodSeconds: 10
          timeoutSeconds: 5
          failureThreshold: 3
        readinessProbe:
          exec:
            command:
            - bash
            - /etc/mongo/readinessProbe.sh
          initialDelaySeconds: 60
          periodSeconds: 2
          timeoutSeconds: 1
          failureThreshold: 3
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: FallbackToLogsOnError # File
        volumeMounts:
          - name: initstate
            mountPath: /etc/mongo/init
            readOnly: false
          - name: runstate
            mountPath: /etc/mongo
            readOnly: false
          - name: dbdata
            mountPath: /var/lib/mongodb
            readOnly: false

      volumes:
      - name: initstate
        emptyDir: {}
      - name: runstate
        emptyDir: {}
      - name: initcontainer
        configMap:
          name: initcontainer
      - name: runcontainer
        configMap:
          name: runcontainer
      - name: podinfo
        downwardAPI:
          items:
            - path: "labels"
              fieldRef:
                fieldPath: metadata.labels
            - path: "annotations"
              fieldRef:
                fieldPath: metadata.annotations
            - path: "pod_name"
              fieldRef:
                fieldPath: metadata.name
            - path: "pod_namespace"
              fieldRef:
                fieldPath: metadata.namespace

  volumeClaimTemplates:
    - metadata:
        name: dbdata
        annotations:
          volume.beta.kubernetes.io/storage-class: local
      spec:
        storageClassName: local
        accessModes: [ "ReadWriteOnce", "ReadWriteMany" ]
        resources:
          requests:
            storage: 500m
        selector:
          matchLabels:
            bind: mongo