webplatform/ops#73 Creating task scripts

use-case:

Set in place a system where we can use salt to achieve maintenance tasks
without writing secrets in target machine, but instead use the pillar system.

First task: send offsite (to DreamObjects) our backups

backup/master.sls

@@ -3,7 +3,6 @@ include:
   - backup
   - rsync.secret
   - cron
-  - webplatform.swift-dreamobjects
 
 backup_location:
   file.directory:
@@ -46,22 +45,6 @@ backup-salt-master:
       - file: /etc/backup.secret
       - file: {{ deploymentBackupMountpoint }}
 
-#swift-upload:
-#  file.managed:
-#    - source: salt://backup/swift-upload.sh
-#    - mode: 755
-#    - name: /usr/local/sbin/wpd-swift-upload.sh
-#  cron.present:
-#    - identifier: swift-upload
-#    - user: root
-#    - minute: random
-#    - hour: 4
-#    - name: 'JOBNAME=swift-upload cronhelper.sh /usr/local/sbin/wpd-swift-upload.sh'
-#    - require:
-#      - file: /usr/bin/cronhelper.sh
-#      - file: /etc/profile.d/swift-dreamobjects.sh
-#      - file: /usr/local/sbin/wpd-swift-upload.sh
-
 rsync-masterdb-backup:
   cron.present:
     - identifier: rsync-masterdb-backup

elasticsearch/init.sls

@@ -86,36 +86,13 @@ chown -R dhc-user:dhc-user {{ es_folder }}:
       - pkg: elasticsearch
 {% endfor %}
 
-#  file.managed:
-#    - source: salt://elasticsearch/files/etc/elasticsearch/elasticsearch.yml.jinja
-#    - template: jinja
-#    - user: elasticsearch
-#    - group: elasticsearch
-#    - mode: 664
-#    - required:
-#      - pkg: elasticsearch
-
 /etc/default/elasticsearch:
   file.exists
-#  file.managed:
-#    - source: salt://elasticsearch/files/etc/default/elasticsearch.jinja
-#    - template: jinja
-#    - required:
-#      - pkg: elasticsearch
-#
 
 /etc/elasticsearch/logging.yml:
   file.exists:
     - require:
       - pkg: elasticsearch
-#  file.managed:
-#    - source: salt://etc/elasticsearch/logging.yml.jinja
-#    - template: jinja
-#    - user: elasticsearch
-#    - group: elasticsearch
-#    - mode: 664
-#    - required:
-#      - pkg: elasticsearch
 
 /etc/monit/conf.d/elasticsearch.conf:
   file.managed:

roles/masterdb.sls

@@ -4,7 +4,6 @@
 
 include:
   - backup.db
-  - webplatform.swift-dreamobjects
   - mysql.server
 
 #

tasks/dreamobjects_upload/apps/discourse.sls

@@ -0,0 +1,18 @@
+{%- set endpoint = salt['pillar.get']('accounts:swift:dreamhost:endpoint') -%}
+{%- set key = salt['pillar.get']('dreamobjects:discourse:key', 'missing_key') -%}
+{%- set secret = salt['pillar.get']('dreamobjects:discourse:secret', 'missing_secret') -%}
+{%- set bucket = salt['pillar.get']('dreamobjects:discourse:bucket', 'missing_bucket') -%}
+{%- set dir = salt['pillar.get']('dreamobjects:discourse:dir', 'missing_dir') -%}
+
+/usr/local/sbin/dreamobjects_uploader.sh:
+  cmd.wait:
+    - onlyif: test -d {{ dir }}
+    - cwd: {{ dir }}
+    - stateful: True
+    - env:
+      - 'ST_KEY': '{{ secret }}'
+      - 'ST_USER': '{{ key }}'
+      - 'ST_AUTH': '{{ endpoint }}'
+      - 'SYNC_DIR': '{{ dir }}'
+      - 'SYNC_BUCKET': '{{ bucket }}'
+

tasks/dreamobjects_upload/docker.sls

@@ -0,0 +1,3 @@
+include:
+  - tasks.dreamobjects_upload.apps.discourse
+

tasks/dreamobjects_upload/init.sls

@@ -0,0 +1,10 @@
+{%- set rolesDict = salt['grains.get']('roles') -%}
+
+
+include:
+{%- if rolesDict|length() >= 1 %}
+{% for roleName in rolesDict %}
+  - tasks.dreamobjects_upload.{{ roleName }}
+{% endfor %}
+{% endif %}
+

top.sls

@@ -30,7 +30,7 @@ base:
     - rsync
     - python
     - python.mysqldb
-    - webplatform.swift-dreamobjects
+    - webplatform.dreamobjects
 #    - backup.salt_master
 #    - backup.mediawiki_xml
     - logging.syslog_ng
@@ -53,6 +53,7 @@ base:
     - backup.master
     - backup.nfs
     - backup.elasticsearch
+    - webplatform.dreamobjects
   'blog*':
     - php
     - nutcracker
@@ -132,6 +133,10 @@ base:
     - match: grain
     - docker
     - webplatform.docker
+    - webplatform.dreamobjects
+  'roles:masterdb':
+    - match: grain
+    - webplatform.dreamobjects
   'roles:frontend':
     - match: grain
     - fxa.frontend

webplatform/swift-dreamobjects.sls → webplatform/dreamobjects.sls

@@ -8,6 +8,11 @@ python-swiftclient-install:
     - require:
       - pkg: python-swiftclient-install
 
+/usr/local/sbin/dreamobjects_uploader.sh:
+  file.managed:
+    - source: salt://webplatform/files/dreamobjects_uploader.sh
+    - mode: 755
+
 /etc/profile.d/swift-dreamobjects.sh:
   file.managed:
     - mode: 755

webplatform/files/dreamobjects_uploader.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+set -e
+
+if [ ! -d "${SYNC_DIR}" ]; then
+    echo "Directory ${SYNC_DIR} doesnt exist"
+    exit 1
+fi
+echo "Directory will be ${SYNC_DIR}"
+
+
+if [ -z "${SYNC_BUCKET}" ]; then
+    echo "Required environment variable SYNC_BUCKET missing"
+    exit 1
+fi
+echo "Bucket will be ${SYNC_BUCKET}"
+
+
+if [ -z "${ST_AUTH}" ]; then
+    echo "Required environment variable ST_AUTH missing"
+    exit 1
+fi
+echo "Endpoint will be ${ST_AUTH}"
+
+
+if [ -z "${ST_KEY}" ]; then
+    echo "Required environment variable ST_KEY missing"
+    exit 1
+fi
+echo "Key will be ${ST_KEY}"
+
+
+if [ -z "${ST_USER}" ]; then
+    echo "Required environment variable ST_USER missing"
+    exit 1
+fi
+echo "User will be ${ST_USER}"
+
+
+BIN=`which swift`
+
+if [ -z "${BIN}" ]; then
+    echo "Python Swift client (python-swiftclient) not found"
+    exit 1
+fi
+
+cd ${SYNC_DIR}
+find ${SYNC_DIR} -type f -mtime -1 | sed 's/^\.\///' | sort -u > dreamobjects_upload.${SYNC_BUCKET}.list
+while read FILE; do swift upload ${SYNC_BUCKET} $FILE --changed; done < dreamobjects_upload.${SYNC_BUCKET}.list
+

webplatform/files/wpd_aliases.sh

@@ -67,6 +67,10 @@ wpd-html-compact () {
   fi
 }
 
+wpd-serve-http () {
+  python -m SimpleHTTPServer 8080
+}
+
 wpd-convert-yml-json () {
   if [ -f "$1" ]; then
     python -c 'import sys, yaml, json; json.dump(yaml.load(sys.stdin), sys.stdout, indent=2)' < $1 > $1.json