Merge pull request #442 from kartoza/develop

Merge develop to master

.gitignore

@@ -61,3 +61,10 @@ venv
 .vagrant
 *.bak
 .idea
+
+# Working data not intended to be versioned
+data_for_import
+scrap.html
+.pycharm_helpers
+.sudo_as_admin_successful
+

REQUIREMENTS.txt

@@ -1,6 +1,6 @@
 #DateUtils==0.6.6
 Django==1.6.0
-GDAL
+#GDAL
 #PIL==1.1.7
 pillow
 Pygments==1.4

SETUP.md

@@ -34,6 +34,14 @@ python setup.py build_ext --include-dirs=/usr/include/gdal/
 pip install --no-download GDAL
 ```
 
+Installing uno
+```
+apt-get install python-uno
+# copy system install over to venv
+.../venv/lib/python2.7/site-packages ln -s /usr/lib/python2.7/dist-packages/uno.py
+.../venv/lib/python2.7/site-packages ln -s /usr/lib/python2.7/dist-packages/unohelper.py
+```
+
 
 # Development database configuration
 

deployment/Makefile

@@ -0,0 +1,268 @@
+PROJECT_ID := catalogue
+
+SHELL := /bin/bash
+
+# ----------------------------------------------------------------------------
+#    P R O D U C T I O N     C O M M A N D S
+# ----------------------------------------------------------------------------
+default: web
+run: build permissions web migrate collectstatic check-permission
+
+deploy: run
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Bringing up fresh instance "
+	@echo "You can access it on http://localhost:62080"
+	@echo "------------------------------------------------------------------"
+
+build:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Building in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) build uwsgi
+
+web:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Running in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) up -d web
+	@# Dont confuse this with the dbbackup make command below
+	@# This one runs the postgis-backup cron container
+	@# We add --no-recreate so that it does not destroy & recreate the db container
+	@docker-compose -p $(PROJECT_ID) up --no-recreate --no-deps -d dbbackups
+
+permissions:
+	# Probably we want something more granular here....
+	# Your sudo password will be needed to set the file permissions
+	# on logs, media, static and pg dirs
+	@if [ ! -d "logs" ]; then mkdir logs; fi
+	@if [ ! -d "media" ]; then mkdir media; fi
+	@if [ ! -d "static" ]; then mkdir static; fi
+	@if [ ! -d "backups" ]; then mkdir backups; fi
+	@if [ -d "logs" ]; then sudo chmod -R a+rwx logs; fi
+	@if [ -d "media" ]; then sudo chmod -R a+rwx media; fi
+	@if [ -d "static" ]; then sudo chmod -R a+rwx static; fi
+	@if [ -d "pg" ]; then sudo chmod -R a+rwx pg; fi
+	@if [ -d "backups" ]; then sudo chmod -R a+rwx backups; fi
+
+db:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Running db in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) up -d db
+
+nginx:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Running nginx in production mode"
+	@echo "Normally you should use this only for testing"
+	@echo "In a production environment you will typically use nginx running"
+	@echo "on the host rather if you have a multi-site host."
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) up -d nginx
+	@echo "Site should now be available at http://localhost"
+
+migrate:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Running migrate static in production mode"
+	@echo "------------------------------------------------------------------"
+	@#http://stackoverflow.com/questions/29689365/auth-user-error-with-django-1-8-and-syncdb-migrate
+	@#and
+	@#http://stackoverflow.com/questions/3143635/how-to-ignore-mv-error
+	@# We add the '-' prefix to the next line as the migration may fail
+	@# but we want to continue anyway.
+	@#We need to migrate accounts first as it has a reference to user model
+	-@docker-compose -p $(PROJECT_ID) run uwsgi python manage.py migrate auth
+	@docker-compose -p $(PROJECT_ID) run uwsgi python manage.py migrate
+
+update-migrations:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Running update migrations in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) run uwsgi python manage.py makemigrations
+
+collectstatic:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Collecting static in production mode"
+	@echo "------------------------------------------------------------------"
+	#@docker-compose -p $(PROJECT_ID) run uwsgi python manage.py collectstatic --noinput
+	#We need to run collect static in the same context as the running
+	# uwsgi container it seems so I use docker exec here
+	@docker exec -t -i $(PROJECT_ID)-uwsgi python manage.py collectstatic --noinput
+
+reload:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Reload django project in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker exec -t -i $(PROJECT_ID)-uwsgi uwsgi --reload  /tmp/django.pid
+
+kill:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Killing in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) kill
+
+rm: kill
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Removing production instance!!! "
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) rm
+
+logs:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Showing uwsgi logs in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) logs uwsgi
+
+dblogs:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Showing db logs in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) logs db
+
+nginxlogs:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Showing nginx logs in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) logs web
+
+shell:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Shelling in in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) run uwsgi /bin/bash
+
+superuser:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Creating a superuser in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) run uwsgi python manage.py createsuperuser
+
+dbshell:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Shelling in in production database"
+	@echo "------------------------------------------------------------------"
+	@docker exec -t -i $(PROJECT_ID)-db psql -U docker -h localhost gis
+
+dbbash:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Bashing in to production database"
+	@echo "------------------------------------------------------------------"
+	@docker exec -t -i $(PROJECT_ID)-db /bin/bash
+
+dbrestore:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Restore dump from backups/latest.dmp in production mode"
+	@echo "------------------------------------------------------------------"
+	@# - prefix causes command to continue even if it fails
+	-@docker exec -t -i $(PROJECT_ID)-db su - postgres -c "dropdb gis"
+	@docker exec -t -i $(PROJECT_ID)-db su - postgres -c "createdb -O docker -T template_postgis gis"
+	@docker exec -t -i $(PROJECT_ID)-db pg_restore /backups/latest.dmp | docker exec -i $(PROJECT_ID)-db su - postgres -c "psql gis"
+
+dbbackup:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Create `date +%d-%B-%Y`.dmp in production mode"
+	@echo "Warning: backups/latest.dmp will be replaced with a symlink to "
+	@echo "the new backup."
+	@echo "------------------------------------------------------------------"
+	@# - prefix causes command to continue even if it fails
+	@# Explicitly don't use -t so we can call this make target over a remote ssh session
+	@docker exec -i $(PROJECT_ID)-db-backups /backups.sh
+	@docker exec -i $(PROJECT_ID)-db-backups cat /var/log/cron.log | tail -2 | head -1 | awk '{print $4}'
+	-@if [ -f "backups/latest.dmp" ]; then rm backups/latest.dmp; fi
+	# backups is intentionally missing from front of first clause below otherwise symlink comes
+	# out with wrong path...
+	@ln -s `date +%Y`/`date +%B`/PG_$(PROJECT_ID)_gis.`date +%d-%B-%Y`.dmp backups/latest.dmp
+	@echo "Backup should be at: backups/`date +%Y`/`date +%B`/PG_$(PROJECT_ID)_gis.`date +%d-%B-%Y`.dmp"
+
+sentry:
+	@echo
+	@echo "--------------------------"
+	@echo "Running sentry production mode"
+	@echo "--------------------------"
+	@docker-compose  -p $(PROJECT_ID) up -d sentry
+
+maillogs:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Showing smtp logs in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker exec -t -i $(PROJECT_ID)-smtp tail -f /var/log/mail.log
+
+mailerrorlogs:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Showing smtp error logs in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker exec -t -i $(PROJECT_ID)-smtp tail -f /var/log/mail.err
+
+mediasync:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Syncing media files from live server"
+	@echo "------------------------------------------------------------------"
+	@rsync -av --progress inasafe-docker:/home/data/changelog.inasafe.org/deployment/media/ media
+
+dbsync:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Syncing database files from live server"
+	@echo "------------------------------------------------------------------"
+	-@ssh inasafe-docker "cd /home/data/changelog.inasafe.org/deployment; make dbbackup"
+	@rsync -av --progress inasafe-docker:/home/data/changelog.inasafe.org/deployment/backups/ backups
+
+worker:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Running worker production mode"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) up -d worker
+
+workerlogs:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Showing celery worker logs in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) logs worker
+
+# ----------------------------------------------------------------------------
+#    DEVELOPMENT C O M M A N D S
+# --no-deps will attach to prod deps if running
+# after running you will have ssh and web ports open (see dockerfile for no's)
+# and you can set your pycharm to use the python in the container
+# Note that pycharm will copy in resources to the /root/ user folder
+# for pydevd etc. If they dont get copied, restart pycharm...
+# ----------------------------------------------------------------------------
+
+devweb: db
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Running in DEVELOPMENT mode"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) up --no-deps -d devweb
+
+check-permission:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Fix permission query error. "
+	@echo "Happens when userena get’s implemented in an already existing project"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) run uwsgi python manage.py check_permissions