Merge pull request #1 from stakater-docker/init-structure

update README and add Dockerfile manifest and run.sh files

Dockerfile

@@ -0,0 +1,13 @@
+FROM phusion/baseimage:0.9.19
+
+RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 &&\
+        echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list && \
+        apt-get -y update && \
+        apt-get -y install mongodb-org cron awscli
+
+ENV CRON_TIME="0 */1 * * *" \
+  TZ=US/Eastern \
+  CRON_TZ=US/Eastern
+
+ADD run.sh /run.sh
+CMD /run.sh

Jenkinsfile

@@ -0,0 +1,7 @@
+#!/usr/bin/env groovy
+@Library('github.com/stakater/[email protected]') _
+
+pushDockerImage {
+    dockerRepositoryURL = "docker.io"
+    imagePrefix = "5.7"
+}

README.md

@@ -1,2 +1,81 @@
 # mongodb-restore-backup
-A repo containing image to backup & restore Mongodb snapshots
+A repo containing image to backup & restore mongodb snapshots
+
+## Usage
+### Running as a single container
+```
+docker run -d \
+  --env AWS_ACCESS_KEY_ID=awsaccesskeyid \
+  --env AWS_SECRET_ACCESS_KEY=awssecretaccesskey \
+  --env BUCKET=s3bucket
+  --env BACKUP_FOLDER=a/sub/folder/path/ \
+  --env INIT_BACKUP=true \
+  --env MONGODB_HOST=mongodb.host \
+  --env MONGODB_PORT=27017 \
+  --env MONGODB_USER=admin \
+  --env MONGODB_PASS=password \
+  stakater/mongo-backup-restore-s3:0.0.4
+```
+
+If you link `stakater/mongo-backup-restore-s3` to a mongodb container with an alias named mongodb, this image will try to auto load the `host`, `port`, `user`, `pass` if possible. Like this:
+
+```
+docker run -d \
+  --env AWS_ACCESS_KEY_ID=myaccesskeyid \
+  --env AWS_SECRET_ACCESS_KEY=mysecretaccesskey \
+  --env BUCKET=mybucketname \
+  --env BACKUP_FOLDER=a/sub/folder/path/ \
+  --env INIT_BACKUP=true \
+  --link my_mongo_db:mongodb \
+  stakater/mongo-backup-restore-s3:0.0.4
+```
+### Running in K8s
+
+`mongo-restore.yaml` contains a manifest for `StatefulSet` which includes a pod running two containers. The first container is the actual mongodb server that is used by the application and the second is the backup sidecar container. This manifest can be used for any deployment of mongo db instance which includes the automatic backup/restore capabilities. By default AWS credentials and bucket name is fetched from the secret named `mongo`
+
+To use it in k8s configure the variables (in Secrets or manifest) and use the following command to deploy:
+
+`kubectl apply -f mongo-restore.yaml -n <namespace>` 
+
+### ENVIRONMENT variables
+
+`AWS_ACCESS_KEY_ID` - your aws access key id (for your s3 bucket)
+
+`AWS_SECRET_ACCESS_KEY`: - your aws secret access key (for your s3 bucket)
+
+`BUCKET`: - your s3 bucket
+
+`BACKUP_FOLDER`: - name of folder or path to put backups (eg `myapp/db_backups/`). defaults to root of bucket.
+
+`MONGODB_HOST` - the host/ip of your mongodb database
+
+`MONGODB_PORT` - the port number of your mongodb database
+
+`MONGODB_USER` - the username of your mongodb database. If MONGODB_USER is empty while MONGODB_PASS is not, the image will use admin as the default username
+
+`MONGODB_PASS` - the password of your mongodb database
+
+`MONGODB_DB` - the database name to dump. If not specified, it will dump all the databases
+
+`EXTRA_OPTS` - any extra options to pass to mongodump command
+
+`CRON_TIME` - the interval of cron job to run mongodump. `0 1 * * *` by default, which is every hour at 0 mins like 1:00, 2:00...
+
+`TZ` - timezone. default: `US/Eastern`
+
+`CRON_TZ` - cron timezone. default: `US/Eastern`
+
+`INIT_BACKUP` - if defined, create a backup when the container launched
+
+`INIT_RESTORE` - if defined, restore from latest when container is launched
+
+`DISABLE_CRON` - if defined, it will skip setting up automated backups. good for when you want to use this container to seed a dev environment.
+
+
+## Acknowledgements
+
+* forked from [halvves](https://github.com/halvves/mongodb-backup-s3)
+
+## Note
+
+This repository is not created through terraform

mongo-restore.yaml

@@ -0,0 +1,102 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: mongodb
+  namespace: pliro-dev
+spec:
+  serviceName: "mongodb"
+  selector:
+    matchLabels:
+      app: mongodb
+  replicas: 1 
+  template:
+    metadata:
+      labels:
+        app: mongodb
+    spec:   
+      tolerations:
+      - key: "dedicated"
+        operator: "Equal"
+        value: "app"
+        effect: "NoSchedule"   
+      containers:
+      - image: mongo:3.4
+        name: mongodb
+        ports:
+        - containerPort: 27017
+          name: tcp
+        volumeMounts:
+        - mountPath: /data/db
+          name: mongodb-pvc      
+        resources: {}          
+      - image: stakater/mongo-backup-restore-s3:0.0.4
+        name: mongodb-backup-restore
+        volumeMounts:
+        - mountPath: /backup
+          name: mongodb-pvc    
+        env:
+        - name: AWS_ACCESS_KEY_ID
+          valueFrom:
+            secretKeyRef:
+              name: mongodb
+              key: aws_access_key_id
+        - name: AWS_SECRET_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: mongodb
+              key: aws_secret_access_key
+        - name: BUCKET
+          valueFrom:
+            secretKeyRef:
+              name: mongodb
+              key: bucket
+        - name: BACKUP_FOLDER
+          value: "mongo/"
+        - name: MONGODB_HOST
+          value: "localhost"
+        - name: MONGODB_PORT
+          value: "27017"
+#        - name: MONGODB_USER   Default not set
+#          value: ""
+#        - name: MONGODB_PASS   Default not set
+#          value: ""
+#        - name: MONGODB_DB     Not specified. so dump all dbs
+#          value: ""
+        - name: CRON_TIME
+          value: "0 */1 * * *"
+        - name: TZ
+          value: "US/Eastern"
+        - name: CRON_TZ
+          value: "US/Eastern"
+        - name: INIT_BACKUP
+          value: "true"
+        - name: INIT_RESTORE
+          value: "true"
+#        - name: DISABLE_CRON
+#          value: "false"
+      restartPolicy: Always
+
+  volumeClaimTemplates:
+  - metadata:
+      name: mongodb-pvc
+    spec:
+      accessModes: [ "ReadWriteMany" ]
+      storageClassName: efs
+      resources:
+        requests:
+          storage: 2Gi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: mongodb
+  name: mongodb-svc
+  namespace: pliro-dev
+spec:
+  ports:
+  - name: "mongo-port"
+    port: 27017
+    targetPort: 27017
+  selector:
+    app: mongodb

run.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+
+MONGODB_HOST=${MONGODB_PORT_27017_TCP_ADDR:-${MONGODB_HOST}}
+MONGODB_HOST=${MONGODB_PORT_1_27017_TCP_ADDR:-${MONGODB_HOST}}
+MONGODB_PORT=${MONGODB_PORT_27017_TCP_PORT:-${MONGODB_PORT}}
+MONGODB_PORT=${MONGODB_PORT_1_27017_TCP_PORT:-${MONGODB_PORT}}
+MONGODB_USER=${MONGODB_USER:-${MONGODB_ENV_MONGODB_USER}}
+MONGODB_PASS=${MONGODB_PASS:-${MONGODB_ENV_MONGODB_PASS}}
+
+S3PATH="s3://$BUCKET/$BACKUP_FOLDER"
+
+[[ ( -z "${MONGODB_USER}" ) && ( -n "${MONGODB_PASS}" ) ]] && MONGODB_USER='admin'
+
+[[ ( -n "${MONGODB_USER}" ) ]] && USER_STR=" --username ${MONGODB_USER}"
+[[ ( -n "${MONGODB_PASS}" ) ]] && PASS_STR=" --password '${MONGODB_PASS}'"
+[[ ( -n "${MONGODB_DB}" ) ]] && DB_STR=" --db ${MONGODB_DB}"
+
+# Export AWS Credentials into env file for cron job
+printenv | sed 's/^\([a-zA-Z0-9_]*\)=\(.*\)$/export \1="\2"/g' | grep -E "^export AWS" > /root/project_env.sh
+
+echo "=> Creating backup script"
+rm -f /backup.sh
+cat <<EOF >> /backup.sh
+#!/bin/bash
+TIMESTAMP=\`/bin/date +"%Y%m%dT%H%M%S"\`
+BACKUP_NAME=\${TIMESTAMP}.dump.gz
+S3BACKUP=${S3PATH}\${BACKUP_NAME}
+S3LATEST=${S3PATH}latest.dump.gz
+echo "=> Backup started"
+if mongodump --host ${MONGODB_HOST} --port ${MONGODB_PORT} ${USER_STR}${PASS_STR}${DB_STR} --archive=\${BACKUP_NAME} --gzip ${EXTRA_OPTS} && aws s3 cp \${BACKUP_NAME} \${S3BACKUP} && aws s3 cp \${S3BACKUP} \${S3LATEST} && rm \${BACKUP_NAME} ;then
+    echo "   > Backup succeeded"
+else
+    echo "   > Backup failed"
+fi
+echo "=> Done"
+EOF
+chmod +x /backup.sh
+echo "=> Backup script created"
+
+echo "=> Creating restore script"
+rm -f /restore.sh
+cat <<EOF >> /restore.sh
+#!/bin/bash
+if [[( -n "\${1}" )]];then
+    RESTORE_ME=\${1}.dump.gz
+else
+    RESTORE_ME=latest.dump.gz
+fi
+S3RESTORE=${S3PATH}\${RESTORE_ME}
+echo "=> Restore database from \${RESTORE_ME}"
+if aws s3 cp \${S3RESTORE} \${RESTORE_ME} && mongorestore --host ${MONGODB_HOST} --port ${MONGODB_PORT} ${USER_STR}${PASS_STR}${DB_STR} --drop ${EXTRA_OPTS} --archive=\${RESTORE_ME} --gzip && rm \${RESTORE_ME}; then
+    echo "   Restore succeeded"
+else
+    echo "   Restore failed"
+fi
+echo "=> Done"
+EOF
+chmod +x /restore.sh
+echo "=> Restore script created"
+
+echo "=> Creating list script"
+rm -f /listbackups.sh
+cat <<EOF >> /listbackups.sh
+#!/bin/bash
+aws s3 ls ${S3PATH}
+EOF
+chmod +x /listbackups.sh
+echo "=> List script created"
+
+ln -s /restore.sh /usr/bin/restore
+ln -s /backup.sh /usr/bin/backup
+ln -s /listbackups.sh /usr/bin/listbackups
+
+touch /mongo_backup.log
+
+if [ -n "${INIT_BACKUP}" ]; then
+    echo "=> Create a backup on the startup"
+    /backup.sh
+fi
+
+if [ -n "${INIT_RESTORE}" ]; then
+    echo "=> Restore store from lastest backup on startup"
+    /restore.sh
+fi
+
+if [ -z "${DISABLE_CRON}" ]; then
+    echo "${CRON_TIME} . /root/project_env.sh; /backup.sh >> /mongo_backup.log 2>&1" > /crontab.conf
+    crontab  /crontab.conf
+    echo "=> Running cron job"
+    cron && tail -f /mongo_backup.log
+fi
+