diff --git a/backup/master.sls b/backup/master.sls index 8386a65..add71a3 100644 --- a/backup/master.sls +++ b/backup/master.sls @@ -3,7 +3,6 @@ include: - backup - rsync.secret - cron - - webplatform.swift-dreamobjects backup_location: file.directory: @@ -46,22 +45,6 @@ backup-salt-master: - file: /etc/backup.secret - file: {{ deploymentBackupMountpoint }} -#swift-upload: -# file.managed: -# - source: salt://backup/swift-upload.sh -# - mode: 755 -# - name: /usr/local/sbin/wpd-swift-upload.sh -# cron.present: -# - identifier: swift-upload -# - user: root -# - minute: random -# - hour: 4 -# - name: 'JOBNAME=swift-upload cronhelper.sh /usr/local/sbin/wpd-swift-upload.sh' -# - require: -# - file: /usr/bin/cronhelper.sh -# - file: /etc/profile.d/swift-dreamobjects.sh -# - file: /usr/local/sbin/wpd-swift-upload.sh - rsync-masterdb-backup: cron.present: - identifier: rsync-masterdb-backup diff --git a/backup/swift-upload.sh b/backup/swift-upload.sh deleted file mode 100644 index a96d5d2..0000000 --- a/backup/swift-upload.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set -e - -source /etc/profile.d/swift-dreamobjects.sh -cd /mnt/backup/ - -FOUND=`find . -type f -mtime -1 | wc -l` - -if [ $FOUND -eq 0 ]; then - echo 'No backup files to upload?' - exit 1 -fi - -find . -type f -mtime -1 -exec swift upload wpd-backups {} \; - - -exit 0 diff --git a/elasticsearch/init.sls b/elasticsearch/init.sls index de9e739..1fbdfbf 100644 --- a/elasticsearch/init.sls +++ b/elasticsearch/init.sls @@ -86,36 +86,13 @@ chown -R dhc-user:dhc-user {{ es_folder }}: - pkg: elasticsearch {% endfor %} -# file.managed: -# - source: salt://elasticsearch/files/etc/elasticsearch/elasticsearch.yml.jinja -# - template: jinja -# - user: elasticsearch -# - group: elasticsearch -# - mode: 664 -# - required: -# - pkg: elasticsearch - /etc/default/elasticsearch: file.exists -# file.managed: -# - source: salt://elasticsearch/files/etc/default/elasticsearch.jinja -# - template: jinja -# - required: -# - pkg: elasticsearch -# /etc/elasticsearch/logging.yml: file.exists: - require: - pkg: elasticsearch -# file.managed: -# - source: salt://etc/elasticsearch/logging.yml.jinja -# - template: jinja -# - user: elasticsearch -# - group: elasticsearch -# - mode: 664 -# - required: -# - pkg: elasticsearch /etc/monit/conf.d/elasticsearch.conf: file.managed: diff --git a/roles/masterdb.sls b/roles/masterdb.sls index 22de8e7..8bba09c 100644 --- a/roles/masterdb.sls +++ b/roles/masterdb.sls @@ -4,7 +4,6 @@ include: - backup.db - - webplatform.swift-dreamobjects - mysql.server # diff --git a/tasks/dreamobjects_upload/apps/discourse.sls b/tasks/dreamobjects_upload/apps/discourse.sls new file mode 100644 index 0000000..cda3ee7 --- /dev/null +++ b/tasks/dreamobjects_upload/apps/discourse.sls @@ -0,0 +1,18 @@ +{%- set endpoint = salt['pillar.get']('accounts:swift:dreamhost:endpoint') -%} +{%- set key = salt['pillar.get']('dreamobjects:discourse:key', 'missing_key') -%} +{%- set secret = salt['pillar.get']('dreamobjects:discourse:secret', 'missing_secret') -%} +{%- set bucket = salt['pillar.get']('dreamobjects:discourse:bucket', 'missing_bucket') -%} +{%- set dir = salt['pillar.get']('dreamobjects:discourse:dir', 'missing_dir') -%} + +/usr/local/sbin/dreamobjects_uploader.sh: + cmd.wait: + - onlyif: test -d {{ dir }} + - cwd: {{ dir }} + - stateful: True + - env: + - 'ST_KEY': '{{ secret }}' + - 'ST_USER': '{{ key }}' + - 'ST_AUTH': '{{ endpoint }}' + - 'SYNC_DIR': '{{ dir }}' + - 'SYNC_BUCKET': '{{ bucket }}' + diff --git a/tasks/dreamobjects_upload/docker.sls b/tasks/dreamobjects_upload/docker.sls new file mode 100644 index 0000000..978de46 --- /dev/null +++ b/tasks/dreamobjects_upload/docker.sls @@ -0,0 +1,3 @@ +include: + - tasks.dreamobjects_upload.apps.discourse + diff --git a/tasks/dreamobjects_upload/init.sls b/tasks/dreamobjects_upload/init.sls new file mode 100644 index 0000000..167fc39 --- /dev/null +++ b/tasks/dreamobjects_upload/init.sls @@ -0,0 +1,10 @@ +{%- set rolesDict = salt['grains.get']('roles') -%} + + +include: +{%- if rolesDict|length() >= 1 %} +{% for roleName in rolesDict %} + - tasks.dreamobjects_upload.{{ roleName }} +{% endfor %} +{% endif %} + diff --git a/top.sls b/top.sls index be53ba2..2b02485 100644 --- a/top.sls +++ b/top.sls @@ -30,7 +30,7 @@ base: - rsync - python - python.mysqldb - - webplatform.swift-dreamobjects + - webplatform.dreamobjects # - backup.salt_master # - backup.mediawiki_xml - logging.syslog_ng @@ -53,6 +53,7 @@ base: - backup.master - backup.nfs - backup.elasticsearch + - webplatform.dreamobjects 'blog*': - php - nutcracker @@ -132,6 +133,10 @@ base: - match: grain - docker - webplatform.docker + - webplatform.dreamobjects + 'roles:masterdb': + - match: grain + - webplatform.dreamobjects 'roles:frontend': - match: grain - fxa.frontend diff --git a/webplatform/swift-dreamobjects.sls b/webplatform/dreamobjects.sls similarity index 87% rename from webplatform/swift-dreamobjects.sls rename to webplatform/dreamobjects.sls index 293a9ce..9d3aea5 100644 --- a/webplatform/swift-dreamobjects.sls +++ b/webplatform/dreamobjects.sls @@ -8,6 +8,11 @@ python-swiftclient-install: - require: - pkg: python-swiftclient-install +/usr/local/sbin/dreamobjects_uploader.sh: + file.managed: + - source: salt://webplatform/files/dreamobjects_uploader.sh + - mode: 755 + /etc/profile.d/swift-dreamobjects.sh: file.managed: - mode: 755 diff --git a/webplatform/files/dreamobjects_uploader.sh b/webplatform/files/dreamobjects_uploader.sh new file mode 100644 index 0000000..85c289a --- /dev/null +++ b/webplatform/files/dreamobjects_uploader.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +set -e + +if [ ! -d "${SYNC_DIR}" ]; then + echo "Directory ${SYNC_DIR} doesnt exist" + exit 1 +fi +echo "Directory will be ${SYNC_DIR}" + + +if [ -z "${SYNC_BUCKET}" ]; then + echo "Required environment variable SYNC_BUCKET missing" + exit 1 +fi +echo "Bucket will be ${SYNC_BUCKET}" + + +if [ -z "${ST_AUTH}" ]; then + echo "Required environment variable ST_AUTH missing" + exit 1 +fi +echo "Endpoint will be ${ST_AUTH}" + + +if [ -z "${ST_KEY}" ]; then + echo "Required environment variable ST_KEY missing" + exit 1 +fi +echo "Key will be ${ST_KEY}" + + +if [ -z "${ST_USER}" ]; then + echo "Required environment variable ST_USER missing" + exit 1 +fi +echo "User will be ${ST_USER}" + + +BIN=`which swift` + +if [ -z "${BIN}" ]; then + echo "Python Swift client (python-swiftclient) not found" + exit 1 +fi + +cd ${SYNC_DIR} +find ${SYNC_DIR} -type f -mtime -1 | sed 's/^\.\///' | sort -u > dreamobjects_upload.${SYNC_BUCKET}.list +while read FILE; do swift upload ${SYNC_BUCKET} $FILE --changed; done < dreamobjects_upload.${SYNC_BUCKET}.list + diff --git a/webplatform/files/wpd_aliases.sh b/webplatform/files/wpd_aliases.sh index 744ebdf..eac139f 100644 --- a/webplatform/files/wpd_aliases.sh +++ b/webplatform/files/wpd_aliases.sh @@ -67,6 +67,10 @@ wpd-html-compact () { fi } +wpd-serve-http () { + python -m SimpleHTTPServer 8080 +} + wpd-convert-yml-json () { if [ -f "$1" ]; then python -c 'import sys, yaml, json; json.dump(yaml.load(sys.stdin), sys.stdout, indent=2)' < $1 > $1.json