Merge

Gruntfile.js

@@ -87,7 +87,8 @@ module.exports = function (grunt) {
                 'crowdsourcing/serializers/*.py',
                 'crowdsourcing/validators/*.py',
                 'crowdsourcing/viewsets/*.py',
-                'crowdsourcing/*.py'
+                'crowdsourcing/*.py',
+                'mturk/*.py'
             ]
         },
 

Procfile

@@ -1 +1,2 @@
 web: bin/start-nginx bin/start-pgbouncer-stunnel uwsgi uwsgi.ini
+worker: bin/start-pgbouncer-stunnel celery -A csp worker -B -q

README.md

@@ -16,7 +16,7 @@ Install [Postgres](http://postgresapp.com/) 9.4+ and create a new database:
     bash> psql
     psql> CREATE DATABASE crowdsource_dev ENCODING 'UTF8';
 
-Create a `local_settings.py` file in the project root folder and configure it to connect to the Postgres database:
+Create a `local_settings.py` file in the project root folder by copying `local_settings_default.py` and configure it to connect to the Postgres database:
 
     DATABASES = {
         "default": {
@@ -25,11 +25,6 @@ Create a `local_settings.py` file in the project root folder and configure it to
         }
     }
 
-    DEBUG = True
-    COMPRESS_OFFLINE = False
-    COMPRESS_ENABLED = False
-    REGISTRATION_ALLOWED = True
-
 Make sure you have [Python](https://www.python.org/downloads/) installed. Test this by opening a command line terminal and typing `python'.
 
 Install [virtualenv](https://virtualenv.pypa.io/en/latest/installation.html) to manage a local setup of your python packages. Go into the directory with the checkout of the code and create the Python virtual environment:
@@ -132,6 +127,8 @@ On subsequent runs, you only need to run:
     vagrant ssh
     python manage.py runserver [::]:8000
 
+# Celery
+To run celery locally: `celery -A csp worker -l info -B`
 
 # Setup with Heroku
 

crowdsourcing/migrations/0068_worker_scope.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9 on 2016-01-22 01:45
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('crowdsourcing', '0067_auto_20160112_2225'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='worker',
+            name='scope',
+            field=models.CharField(default=b'daemo', max_length=32),
+        ),
+    ]

crowdsourcing/models.py

@@ -132,6 +132,7 @@ class Worker(models.Model):
     skills = models.ManyToManyField(Skill, through='WorkerSkill')
     deleted = models.BooleanField(default=False)
     alias = models.CharField(max_length=32, error_messages={'required': "Please enter an alias!"})
+    scope = models.CharField(max_length=32, default='daemo')
 
 
 class WorkerSkill(models.Model):

crowdsourcing/serializers/project.py

@@ -9,6 +9,7 @@
 from crowdsourcing.serializers.message import CommentSerializer
 from crowdsourcing.utils import generate_random_id
 from crowdsourcing.serializers.file import BatchFileSerializer
+from mturk.tasks import mturk_update_status
 
 
 class CategorySerializer(DynamicFieldsModelSerializer):
@@ -159,6 +160,10 @@ def update(self, *args, **kwargs):
         self.instance.name = self.validated_data.get('name', self.instance.name)
         self.instance.price = self.validated_data.get('price', self.instance.price)
         self.instance.repetition = self.validated_data.get('repetition', self.instance.repetition)
+        if status != self.instance.status \
+            and status in (models.Project.STATUS_PAUSED, models.Project.STATUS_IN_PROGRESS) and \
+                self.instance.status in (models.Project.STATUS_PAUSED, models.Project.STATUS_IN_PROGRESS):
+            mturk_update_status.delay({'id': self.instance.id, 'status': status})
         self.instance.status = status
         self.instance.save()
         return self.instance
@@ -169,7 +174,7 @@ def fork(self, *args, **kwargs):
         batch_files = self.instance.batch_files.all()
 
         project = self.instance
-        project.name = project.name + ' (copy)'
+        project.name += ' (copy)'
         project.status = 1
         project.is_prototype = False
         project.parent = models.Project.objects.get(pk=self.instance.id)

crowdsourcing/serializers/task.py

@@ -1,8 +1,10 @@
 from __future__ import division
-from crowdsourcing import models
+
 from rest_framework import serializers
-from crowdsourcing.serializers.dynamic import DynamicFieldsModelSerializer
 from django.db import transaction
+
+from crowdsourcing import models
+from crowdsourcing.serializers.dynamic import DynamicFieldsModelSerializer
 from crowdsourcing.serializers.template import TemplateSerializer
 from crowdsourcing.serializers.message import CommentSerializer
 
@@ -124,6 +126,7 @@ def create(self, **kwargs):
                     skipped = True
                 if len(list(tasks)) and not skipped:
                     task_worker = models.TaskWorker.objects.create(worker=kwargs['worker'], task=tasks[0])
+
                 elif len(list(tasks)) and skipped:
                     task_worker = models.TaskWorker.objects.get(worker=kwargs['worker'], task=tasks[0])
                     task_worker.task_status = 1
@@ -269,7 +272,7 @@ def get_has_comments(obj):
     @staticmethod
     def get_project_data(obj):
         from crowdsourcing.serializers.project import ProjectSerializer
-        project = ProjectSerializer(instance=obj.project, many=False, fields=('id', 'name')).data
+        project = ProjectSerializer(instance=obj.project, many=False, fields=('id', 'name', 'owner')).data
         return project
 
     @staticmethod

crowdsourcing/viewsets/project.py

@@ -202,4 +202,5 @@ def get_preview(self, request, *args, **kwargs):
         import json
         socket_manager = SocketManager()
         socket_manager.broadcast(channel='foo', message=json.dumps({'id': project.id, 'name': project.name}))
+
         return Response(data=task_serializer.data, status=status.HTTP_200_OK)

crowdsourcing/viewsets/task.py

@@ -11,6 +11,7 @@
 from crowdsourcing.permissions.project import IsProjectOwnerOrCollaborator
 from crowdsourcing.models import Task, TaskWorker, TaskWorkerResult
 from crowdsourcing.permissions.task import HasExceededReservedLimit
+from mturk.tasks import mturk_hit_update, mturk_approve
 
 
 class TaskViewSet(viewsets.ModelViewSet):
@@ -105,6 +106,7 @@ def create(self, request, *args, **kwargs):
         serialized_data = {}
         if http_status == 200:
             serialized_data = TaskWorkerSerializer(instance=instance).data
+            mturk_hit_update.delay({'id': instance.task.id})
         return Response(serialized_data, http_status)
 
     def destroy(self, request, *args, **kwargs):
@@ -113,6 +115,7 @@ def destroy(self, request, *args, **kwargs):
         instance, http_status = serializer.create(worker=request.user.userprofile.worker, project=obj.task.project_id)
         obj.task_status = TaskWorker.STATUS_SKIPPED
         obj.save()
+        mturk_hit_update.delay({'id': obj.task.id})
         serialized_data = {}
         if http_status == status.HTTP_200_OK:
             serialized_data = TaskWorkerSerializer(instance=instance).data
@@ -133,6 +136,7 @@ def accept_all(self, request, *args, **kwargs):
         task_workers = TaskWorker.objects.filter(task_status=TaskWorker.STATUS_SUBMITTED, task_id=kwargs['task__id'])
         list_workers = list(chain.from_iterable(task_workers.values_list('id')))
         task_workers.update(task_status=TaskWorker.STATUS_ACCEPTED, last_updated=timezone.now())
+        mturk_approve.delay(list_workers)
         return Response(data=list_workers, status=status.HTTP_200_OK)
 
     @list_route(methods=['get'], url_path='list-my-tasks')
@@ -153,6 +157,8 @@ def list_my_tasks(self, request, *args, **kwargs):
     @list_route(methods=['post'])
     def drop_saved_tasks(self, request, *args, **kwargs):
         task_ids = request.data.get('task_ids', [])
+        for task_id in task_ids:
+            mturk_hit_update.delay({'id': task_id})
         self.queryset.filter(task_id__in=task_ids, worker=request.user.userprofile.worker.id).update(
             task_status=TaskWorker.STATUS_SKIPPED, last_updated=timezone.now())
         return Response('Success', status.HTTP_200_OK)

csp/__init__.py

@@ -0,0 +1,3 @@
+from __future__ import absolute_import
+
+from .celery import app as celery_app

csp/celery.py

@@ -0,0 +1,11 @@
+from __future__ import absolute_import
+import os
+from celery import Celery
+
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'csp.settings')
+app = Celery('csp')
+
+app.config_from_object('django.conf:settings')
+
+app.autodiscover_tasks(['crowdsourcing', 'mturk'])

csp/settings.py

@@ -13,7 +13,7 @@
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 import logging
 import dj_redis_url
-
+from datetime import timedelta
 import os
 import django
 import dj_database_url
@@ -27,7 +27,7 @@
 SECRET_KEY = 'v1*ah#)@vyov!7c@n&c2^-*=8d)-d!u9@#c4o*@k=1(1!jul6&'
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = False
+DEBUG = os.environ.get('DEBUG', False)
 
 TEMPLATE_DEBUG = DEBUG
 APPEND_SLASH = True
@@ -78,7 +78,8 @@
     'rest_framework',
     'oauth2_provider',
     'ws4redis',
-    'crowdsourcing'
+    'crowdsourcing',
+    'mturk'
 )
 
 MIDDLEWARE_CLASSES = (
@@ -98,7 +99,7 @@
 TEMPLATES = [
     {
         'BACKEND': 'django.template.backends.django.DjangoTemplates',
-        'DIRS': [os.path.join(BASE_DIR, 'static/django_templates')],
+        'DIRS': [os.path.join(BASE_DIR, 'static/django_templates'), os.path.join(BASE_DIR, 'static/mturk')],
         'APP_DIRS': True,
         'OPTIONS': {
             'context_processors': [
@@ -197,9 +198,35 @@
 LOGIN_URL = '/login'
 USERNAME_MAX_LENGTH = 30
 
+SITE_HOST = os.environ.get('SITE_HOST', 'https://daemo.herokuapp.com')
+
 REDIS_URL = os.environ.get('REDIS_URL', 'redis://localhost:6379')
 REDIS_CONNECTION = dj_redis_url.parse(REDIS_URL)
 
+# MTurk
+MTURK_CLIENT_ID = os.environ.get('MTURK_CLIENT_ID', 'INVALID')
+MTURK_CLIENT_SECRET = os.environ.get('MTURK_CLIENT_SECRET', 'INVALID')
+MTURK_HOST = 'mechanicalturk.sandbox.amazonaws.com'
+MTURK_HASH_MIN_LENGTH = 8
+MTURK_WORKER_USERNAME = 'mturk'
+MTURK_QUALIFICATIONS = os.environ.get('MTURK_QUALIFICATIONS', True)
+
+# Celery
+BROKER_URL = REDIS_URL
+CELERY_RESULT_BACKEND = REDIS_URL
+CELERY_ACCEPT_CONTENT = ['application/json']
+CELERY_TASK_SERIALIZER = 'json'
+CELERY_RESULT_SERIALIZER = 'json'
+CELERY_TIMEZONE = 'America/Los_Angeles'
+
+CELERYBEAT_SCHEDULE = {
+    'mturk-every-30min': {
+        'task': 'mturk.tasks.mturk_publish',
+        'schedule': timedelta(seconds=60),
+    },
+}
+
+# Sessions
 SESSION_ENGINE = 'redis_sessions.session'
 SESSION_REDIS_HOST = REDIS_CONNECTION['HOST']
 SESSION_REDIS_PORT = REDIS_CONNECTION['PORT']

csp/urls.py

@@ -1,6 +1,7 @@
 from django.conf.urls import patterns, include, url
 from django.contrib.staticfiles.urls import staticfiles_urlpatterns
 from crowdsourcing import views
+from mturk import views as mturk_views
 from crowdsourcing.viewsets.project import *
 from crowdsourcing.viewsets.user import UserViewSet, UserProfileViewSet, UserPreferencesViewSet
 from crowdsourcing.viewsets.requester import RequesterViewSet, QualificationViewSet
@@ -15,8 +16,10 @@
 from crowdsourcing.viewsets.file import FileViewSet
 from crowdsourcing.viewsets.payment import PayPalFlowViewSet, FinancialAccountViewSet
 from rest_framework.routers import SimpleRouter
+from mturk.viewsets import MTurkAssignmentViewSet
 
 router = SimpleRouter(trailing_slash=True)
+mturk_router = SimpleRouter(trailing_slash=False)
 router.register(r'api/profile', UserProfileViewSet)
 router.register(r'api/user', UserViewSet)
 router.register(r'api/preferences', UserPreferencesViewSet)
@@ -43,6 +46,7 @@
 router.register(r'api/payment-paypal', PayPalFlowViewSet)
 router.register(r'api/financial-accounts', FinancialAccountViewSet)
 router.register(r'^api/file', FileViewSet)
+mturk_router.register(r'^api/mturk', MTurkAssignmentViewSet)
 
 urlpatterns = patterns('',
                        url(r'^api/v1/auth/registration-successful', views.registration_successful),
@@ -52,10 +56,12 @@
                        url(r'^api/oauth2-ng/token', views.Oauth2TokenView.as_view()),
                        url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')),
                        url(r'^api/google-drive/init', GoogleDriveOauth.as_view({'post': 'auth_init'})),
+                       url(r'^mturk/task', mturk_views.mturk_index),
                        url(r'^api/google-drive/finish', GoogleDriveOauth.as_view({'post': 'auth_end'})),
                        url(r'^api/google-drive/list-files', GoogleDriveViewSet.as_view({'get': 'query'})),
                        url(r'^api/done/$', ExternalSubmit.as_view()),
                        url(r'', include(router.urls)),
+                       url(r'', include(mturk_router.urls)),
                        url('^.*$', views.home, name='home'),
                        )
 

local_settings_default.py

@@ -1,8 +1,16 @@
 DEBUG = True
+
 COMPRESS_OFFLINE = False
 COMPRESS_ENABLED = False
+
 REGISTRATION_ALLOWED = True
 
+CELERY_RESULT_BACKEND = 'redis://localhost:6379'
+BROKER_URL = 'redis://localhost:6379'
+CELERY_TIMEZONE = 'America/Los_Angeles'
+
+SITE_HOST = 'https://localhost:8000'
+
 DATABASES = {
     "default": {
         "ENGINE": "django.db.backends.postgresql_psycopg2",

mturk/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class MturkConfig(AppConfig):
+    name = 'mturk'