From ee32c693e14dec0cf13f329683b16cf30d698f57 Mon Sep 17 00:00:00 2001 From: Pedro Rodriguez Date: Thu, 23 Apr 2015 17:46:00 -0700 Subject: [PATCH 01/12] style --- ampcrowd/basecrowd/views.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ampcrowd/basecrowd/views.py b/ampcrowd/basecrowd/views.py index 99c9cc7..3d43e55 100644 --- a/ampcrowd/basecrowd/views.py +++ b/ampcrowd/basecrowd/views.py @@ -159,7 +159,7 @@ def get_assignment(request, crowd_name): # Load the template and render it. template = get_scoped_template(crowd_name, current_task.task_type + '.html', - context=context) + context=context) return HttpResponse(template.render(RequestContext(request, context))) @@ -182,7 +182,6 @@ def get_scoped_template(crowd_name, template_name, context=None): @require_POST @csrf_exempt def post_response(request, crowd_name): - # get the interface implementation from the crowd name. interface, model_spec = CrowdRegistry.get_registry_entry(crowd_name) @@ -216,9 +215,9 @@ def post_response(request, crowd_name): # Check if this task has been finished # If we've gotten too many responses, ignore. if (not current_task.is_complete - and current_task.responses.count() >= current_task.num_assignments): + and current_task.responses.count() >= current_task.num_assignments): current_task.is_complete = True current_task.save() gather_answer.delay(current_task.task_id, model_spec) - return HttpResponse('ok') # AJAX call succeded. + return HttpResponse('ok') # AJAX call succeeded. From db68e1e1a6eab20ae4255268da290383057037cb Mon Sep 17 00:00:00 2001 From: Pedro Rodriguez Date: Fri, 24 Apr 2015 08:00:47 -0700 Subject: [PATCH 02/12] js file --- ampcrowd/basecrowd/templates/basecrowd/javascript.html | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 ampcrowd/basecrowd/templates/basecrowd/javascript.html diff --git a/ampcrowd/basecrowd/templates/basecrowd/javascript.html b/ampcrowd/basecrowd/templates/basecrowd/javascript.html new file mode 100644 index 0000000..71ce692 --- /dev/null +++ b/ampcrowd/basecrowd/templates/basecrowd/javascript.html @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file From f9dec35ac9d9460cb4316902eef2b0a52c8b385f Mon Sep 17 00:00:00 2001 From: Pedro Rodriguez Date: Fri, 24 Apr 2015 18:10:49 -0700 Subject: [PATCH 03/12] added django models for Task and Resources along with tests. Created docker script for running tests and modified entrypoint to take argument "test" --- ampcrowd/basecrowd/models.py | 35 ++++++++++++++ ampcrowd/basecrowd/tests.py | 3 -- ampcrowd/basecrowd/tests/test_models.py | 63 +++++++++++++++++++++++++ ampcrowd/docker-entrypoint.sh | 22 +++++---- docker-test.sh | 3 ++ 5 files changed, 113 insertions(+), 13 deletions(-) delete mode 100644 ampcrowd/basecrowd/tests.py create mode 100644 ampcrowd/basecrowd/tests/test_models.py create mode 100755 docker-test.sh diff --git a/ampcrowd/basecrowd/models.py b/ampcrowd/basecrowd/models.py index 1b63bc6..bffd1da 100644 --- a/ampcrowd/basecrowd/models.py +++ b/ampcrowd/basecrowd/models.py @@ -150,3 +150,38 @@ def add_model_rels(self): # responses pertain to a task self.add_rel(self.response_model, self.task_model, models.ForeignKey, 'task', 'responses') + + +class TemplateResource(models.Model): + name = models.CharField(max_length=200) + content = models.TextField() + direct_dependencies = models.ManyToManyField('self', symmetrical=False, related_name="upstream_dependencies") + direct_requirements = models.ManyToManyField('self', symmetrical=False, related_name="upstream_requirements") + + @property + def dependencies(self): + query_set = list(self.direct_dependencies.all()) + all_dependencies = set(query_set) + current_dependencies = list(query_set) + while True: + prior_size = len(all_dependencies) + new_dependencies = set() + for dep in current_dependencies: + deps = set(dep.direct_dependencies.all()) + new_dependencies = new_dependencies.union(deps) + all_dependencies = all_dependencies.union(deps) + if prior_size == len(all_dependencies): + break + else: + current_dependencies = list(new_dependencies) + return all_dependencies + + def __str__(self): + return self.name + + +class TaskType(models.Model): + name = models.CharField(max_length=200) + iterator_template = models.ForeignKey(TemplateResource, related_name='iterator_task') + point_template = models.ForeignKey(TemplateResource, related_name='point_task') + renderer = models.ForeignKey(TemplateResource, related_name='renderer_task') diff --git a/ampcrowd/basecrowd/tests.py b/ampcrowd/basecrowd/tests.py deleted file mode 100644 index 7ce503c..0000000 --- a/ampcrowd/basecrowd/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/ampcrowd/basecrowd/tests/test_models.py b/ampcrowd/basecrowd/tests/test_models.py new file mode 100644 index 0000000..5256112 --- /dev/null +++ b/ampcrowd/basecrowd/tests/test_models.py @@ -0,0 +1,63 @@ +from django.test import TestCase +from basecrowd.models import TaskType, TemplateResource + + +class TasksTestCase(TestCase): + def setUp(self): + html = TemplateResource.objects.create(name="html") + js = TemplateResource.objects.create(name="js") + css = TemplateResource.objects.create(name="css") + bootstrap = TemplateResource.objects.create(name="bootstrap") + bootstrap.direct_dependencies.add(html, js, css) + + jquery = TemplateResource.objects.create(name="jquery") + jquery.direct_dependencies.add(html, js, css) + + reactjs = TemplateResource.objects.create(name="reactjs") + reactjs.direct_dependencies.add(js) + + jqueryui = TemplateResource.objects.create(name="jqueryui") + jqueryui.direct_dependencies.add(jquery) + + bootstrap_react_gallery = TemplateResource.objects.create(name="bootstrap react gallery") + bootstrap_react_gallery.direct_dependencies.add(reactjs, bootstrap) + + monolithic_app = TemplateResource.objects.create(name="monolithic app") + monolithic_app.direct_dependencies.add( + bootstrap_react_gallery, jqueryui, jquery, js, css, html, reactjs + ) + + def test_template_dependencies(self): + html = TemplateResource.objects.get(name="html") + css = TemplateResource.objects.get(name="css") + js = TemplateResource.objects.get(name="js") + jquery = TemplateResource.objects.get(name="jquery") + bootstrap = TemplateResource.objects.get(name="bootstrap") + reactjs = TemplateResource.objects.get(name="reactjs") + jqueryui = TemplateResource.objects.get(name="jqueryui") + bootstrap_react_gallery = TemplateResource.objects.get(name="bootstrap react gallery") + monolithic_app = TemplateResource.objects.get(name="monolithic app") + self.assertSetEqual( + jquery.dependencies, + {html, css, js} + ) + self.assertSetEqual( + html.dependencies, + set() + ) + self.assertSetEqual( + reactjs.dependencies, + {js} + ) + self.assertSetEqual( + jqueryui.dependencies, + {jquery, html, js, css} + ) + self.assertSetEqual( + bootstrap_react_gallery.dependencies, + {html, js, css, reactjs, bootstrap} + ) + self.assertSetEqual( + monolithic_app.dependencies, + {html, js, css, reactjs, jquery, jqueryui, bootstrap, bootstrap_react_gallery} + ) diff --git a/ampcrowd/docker-entrypoint.sh b/ampcrowd/docker-entrypoint.sh index 5dfdd62..9ba09fb 100644 --- a/ampcrowd/docker-entrypoint.sh +++ b/ampcrowd/docker-entrypoint.sh @@ -13,16 +13,10 @@ SSL=0 FOREGROUND=0 while getopts "dsf" OPTION do - case $OPTION in - d) - DEVELOP=1 - ;; - s) - SSL=1 - ;; - f) - FOREGROUND=1 - ;; + case "$OPTION" in + d) DEVELOP=1;; + s) SSL=1;; + f) FOREGROUND=1;; esac done @@ -30,6 +24,14 @@ export DEVELOP export SSL export FOREGROUND +ARG1=${@:OPTIND:1} + +if [ "$ARG1" = "test" ] +then + (cd ampcrowd && python manage.py test **/tests/) + exit +fi + if [ "$DEVELOP" -eq "1" ] then echo "Celery launched in debug mode" diff --git a/docker-test.sh b/docker-test.sh new file mode 100755 index 0000000..270e5a4 --- /dev/null +++ b/docker-test.sh @@ -0,0 +1,3 @@ +docker-compose kill +docker-compose build +docker-compose run --rm --service-ports web test From 045a280cf5249a0d864b87005fdf03dc0e487cd6 Mon Sep 17 00:00:00 2001 From: Pedro Rodriguez Date: Mon, 27 Apr 2015 17:02:11 -0700 Subject: [PATCH 04/12] added placeholder for requirements function --- ampcrowd/basecrowd/models.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ampcrowd/basecrowd/models.py b/ampcrowd/basecrowd/models.py index bffd1da..376b0df 100644 --- a/ampcrowd/basecrowd/models.py +++ b/ampcrowd/basecrowd/models.py @@ -176,6 +176,11 @@ def dependencies(self): current_dependencies = list(new_dependencies) return all_dependencies + @property + def requirements(self): + pass + + def __str__(self): return self.name From 957db076fd47fdccd318fa50c885c0f55bfa7b1e Mon Sep 17 00:00:00 2001 From: Pedro Rodriguez Date: Tue, 28 Apr 2015 08:56:02 -0700 Subject: [PATCH 05/12] deleted old dashboard code and replaced it with a placeholder boostrap template with react imports. Will work on adding new general dashboard now --- .../results_dashboard/static/dashboard.css | 105 +++ ampcrowd/results_dashboard/static/holder.js | 12 + .../templates/results_dashboard/index.html | 728 ++++++------------ 3 files changed, 358 insertions(+), 487 deletions(-) create mode 100644 ampcrowd/results_dashboard/static/dashboard.css create mode 100644 ampcrowd/results_dashboard/static/holder.js diff --git a/ampcrowd/results_dashboard/static/dashboard.css b/ampcrowd/results_dashboard/static/dashboard.css new file mode 100644 index 0000000..e0e3632 --- /dev/null +++ b/ampcrowd/results_dashboard/static/dashboard.css @@ -0,0 +1,105 @@ +/* + * Base structure + */ + +/* Move down content because we have a fixed navbar that is 50px tall */ +body { + padding-top: 50px; +} + + +/* + * Global add-ons + */ + +.sub-header { + padding-bottom: 10px; + border-bottom: 1px solid #eee; +} + +/* + * Top navigation + * Hide default border to remove 1px line. + */ +.navbar-fixed-top { + border: 0; +} + +/* + * Sidebar + */ + +/* Hide for mobile, show later */ +.sidebar { + display: none; +} +@media (min-width: 768px) { + .sidebar { + position: fixed; + top: 51px; + bottom: 0; + left: 0; + z-index: 1000; + display: block; + padding: 20px; + overflow-x: hidden; + overflow-y: auto; /* Scrollable contents if viewport is shorter than content. */ + background-color: #f5f5f5; + border-right: 1px solid #eee; + } +} + +/* Sidebar navigation */ +.nav-sidebar { + margin-right: -21px; /* 20px padding + 1px border */ + margin-bottom: 20px; + margin-left: -20px; +} +.nav-sidebar > li > a { + padding-right: 20px; + padding-left: 20px; +} +.nav-sidebar > .active > a, +.nav-sidebar > .active > a:hover, +.nav-sidebar > .active > a:focus { + color: #fff; + background-color: #428bca; +} + + +/* + * Main content + */ + +.main { + padding: 20px; +} +@media (min-width: 768px) { + .main { + padding-right: 40px; + padding-left: 40px; + } +} +.main .page-header { + margin-top: 0; +} + + +/* + * Placeholder dashboard ideas + */ + +.placeholders { + margin-bottom: 30px; + text-align: center; +} +.placeholders h4 { + margin-bottom: 0; +} +.placeholder { + margin-bottom: 20px; +} +.placeholder img { + display: inline-block; + border-radius: 50%; +} diff --git a/ampcrowd/results_dashboard/static/holder.js b/ampcrowd/results_dashboard/static/holder.js new file mode 100644 index 0000000..a04ecff --- /dev/null +++ b/ampcrowd/results_dashboard/static/holder.js @@ -0,0 +1,12 @@ +/*! + +Holder - client side image placeholders +Version 2.4.1+f2l1h +© 2014 Ivan Malopinsky - http://imsky.co + +Site: http://imsky.github.io/holder +Issues: https://github.com/imsky/holder/issues +License: http://opensource.org/licenses/MIT + +*/ +!function(e,t,r){t[e]=r}("onDomReady",this,function(e){"use strict";function t(e){if(!b){if(!a.body)return i(t);for(b=!0;e=S.shift();)i(e)}}function r(e){(y||e.type===s||a[c]===u)&&(n(),t())}function n(){y?(a[x](m,r,d),e[x](s,r,d)):(a[g](v,r),e[g](h,r))}function i(e,t){setTimeout(e,+t>=0?t:1)}function o(e){b?i(e):S.push(e)}null==document.readyState&&document.addEventListener&&(document.addEventListener("DOMContentLoaded",function E(){document.removeEventListener("DOMContentLoaded",E,!1),document.readyState="complete"},!1),document.readyState="loading");var a=e.document,l=a.documentElement,s="load",d=!1,h="on"+s,u="complete",c="readyState",f="attachEvent",g="detachEvent",p="addEventListener",m="DOMContentLoaded",v="onreadystatechange",x="removeEventListener",y=p in a,w=d,b=d,S=[];if(a[c]===u)i(t);else if(y)a[p](m,r,d),e[p](s,r,d);else{a[f](v,r),e[f](h,r);try{w=null==e.frameElement&&l}catch(C){}w&&w.doScroll&&!function k(){if(!b){try{w.doScroll("left")}catch(e){return i(k,50)}n(),t()}}()}return o.version="1.4.0",o.isReady=function(){return b},o}(this)),document.querySelectorAll||(document.querySelectorAll=function(e){var t,r=document.createElement("style"),n=[];for(document.documentElement.firstChild.appendChild(r),document._qsa=[],r.styleSheet.cssText=e+"{x-qsa:expression(document._qsa && document._qsa.push(this))}",window.scrollBy(0,0),r.parentNode.removeChild(r);document._qsa.length;)t=document._qsa.shift(),t.style.removeAttribute("x-qsa"),n.push(t);return document._qsa=null,n}),document.querySelector||(document.querySelector=function(e){var t=document.querySelectorAll(e);return t.length?t[0]:null}),document.getElementsByClassName||(document.getElementsByClassName=function(e){return e=String(e).replace(/^|\s+/g,"."),document.querySelectorAll(e)}),Object.keys||(Object.keys=function(e){if(e!==Object(e))throw TypeError("Object.keys called on non-object");var t,r=[];for(t in e)Object.prototype.hasOwnProperty.call(e,t)&&r.push(t);return r}),function(e){var t="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=";e.atob=e.atob||function(e){e=String(e);var r,n=0,i=[],o=0,a=0;if(e=e.replace(/\s/g,""),e.length%4===0&&(e=e.replace(/=+$/,"")),e.length%4===1)throw Error("InvalidCharacterError");if(/[^+/0-9A-Za-z]/.test(e))throw Error("InvalidCharacterError");for(;n>16&255)),i.push(String.fromCharCode(o>>8&255)),i.push(String.fromCharCode(255&o)),a=0,o=0),n+=1;return 12===a?(o>>=4,i.push(String.fromCharCode(255&o))):18===a&&(o>>=2,i.push(String.fromCharCode(o>>8&255)),i.push(String.fromCharCode(255&o))),i.join("")},e.btoa=e.btoa||function(e){e=String(e);var r,n,i,o,a,l,s,d=0,h=[];if(/[^\x00-\xFF]/.test(e))throw Error("InvalidCharacterError");for(;d>2,a=(3&r)<<4|n>>4,l=(15&n)<<2|i>>6,s=63&i,d===e.length+2?(l=64,s=64):d===e.length+1&&(s=64),h.push(t.charAt(o),t.charAt(a),t.charAt(l),t.charAt(s));return h.join("")}}(this),function(){function e(t,r,n){t.document;var i,o=t.currentStyle[r].match(/([\d\.]+)(%|cm|em|in|mm|pc|pt|)/)||[0,0,""],a=o[1],l=o[2];return n=n?/%|em/.test(l)&&t.parentElement?e(t.parentElement,"fontSize",null):16:n,i="fontSize"==r?n:/width/i.test(r)?t.clientWidth:t.clientHeight,"%"==l?a/100*i:"cm"==l?.3937*a*96:"em"==l?a*n:"in"==l?96*a:"mm"==l?.3937*a*96/10:"pc"==l?12*a*96/72:"pt"==l?96*a/72:a}function t(e,t){var r="border"==t?"Width":"",n=t+"Top"+r,i=t+"Right"+r,o=t+"Bottom"+r,a=t+"Left"+r;e[t]=(e[n]==e[i]&&e[n]==e[o]&&e[n]==e[a]?[e[n]]:e[n]==e[o]&&e[a]==e[i]?[e[n],e[i]]:e[a]==e[i]?[e[n],e[i],e[o]]:[e[n],e[i],e[o],e[a]]).join(" ")}function r(r){var n,i=this,o=r.currentStyle,a=e(r,"fontSize"),l=function(e){return"-"+e.toLowerCase()};for(n in o)if(Array.prototype.push.call(i,"styleFloat"==n?"float":n.replace(/[A-Z]/,l)),"width"==n)i[n]=r.offsetWidth+"px";else if("height"==n)i[n]=r.offsetHeight+"px";else if("styleFloat"==n)i.float=o[n];else if(/margin.|padding.|border.+W/.test(n)&&"auto"!=i[n])i[n]=Math.round(e(r,n,a))+"px";else if(/^outline/.test(n))try{i[n]=o[n]}catch(s){i.outlineColor=o.color,i.outlineStyle=i.outlineStyle||"none",i.outlineWidth=i.outlineWidth||"0px",i.outline=[i.outlineColor,i.outlineWidth,i.outlineStyle].join(" ")}else i[n]=o[n];t(i,"margin"),t(i,"padding"),t(i,"border"),i.fontSize=Math.round(a)+"px"}window.getComputedStyle||(r.prototype={constructor:r,getPropertyPriority:function(){throw new Error("NotSupportedError: DOM Exception 9")},getPropertyValue:function(e){var t=e.replace(/-([a-z])/g,function(e){return e=e.charAt?e.split(""):e,e[1].toUpperCase()}),r=this[t];return r},item:function(e){return this[e]},removeProperty:function(){throw new Error("NoModificationAllowedError: DOM Exception 7")},setProperty:function(){throw new Error("NoModificationAllowedError: DOM Exception 7")},getPropertyCSSValue:function(){throw new Error("NotSupportedError: DOM Exception 9")}},window.getComputedStyle=function(e){return new r(e)})}(),Object.prototype.hasOwnProperty||(Object.prototype.hasOwnProperty=function(e){var t=this.__proto__||this.constructor.prototype;return e in this&&(!(e in t)||t[e]!==this[e])}),function(e,t){e.augment=t()}(this,function(){"use strict";var e=function(){},t=Array.prototype.slice,r=function(r,n){var i=e.prototype="function"==typeof r?r.prototype:r,o=new e,a=n.apply(o,t.call(arguments,2).concat(i));if("object"==typeof a)for(var l in a)o[l]=a[l];if(!o.hasOwnProperty("constructor"))return o;var s=o.constructor;return s.prototype=o,s};return r.defclass=function(e){var t=e.constructor;return t.prototype=e,t},r.extend=function(e,t){return r(e,function(e){return this.uber=e,t})},r}),function(e,t){function r(e,t,r,o){var a=n(r.substr(r.lastIndexOf(e.domain)),e);a&&i(null,o,a,t)}function n(e,t){for(var r={theme:p(A.settings.themes.gray,null),stylesheets:t.stylesheets,holderURL:[]},n=!1,i=String.fromCharCode(11),o=e.replace(/([^\\])\//g,"$1"+i).split(i),a=/%[0-9a-f]{2}/gi,l=o.length,s=0;l>s;s++){var d=o[s];if(d.match(a))try{d=decodeURIComponent(d)}catch(h){d=o[s]}var u=!1;if(A.flags.dimensions.match(d))n=!0,r.dimensions=A.flags.dimensions.output(d),u=!0;else if(A.flags.fluid.match(d))n=!0,r.dimensions=A.flags.fluid.output(d),r.fluid=!0,u=!0;else if(A.flags.textmode.match(d))r.textmode=A.flags.textmode.output(d),u=!0;else if(A.flags.colors.match(d)){var c=A.flags.colors.output(d);r.theme=p(r.theme,c),u=!0}else if(t.themes[d])t.themes.hasOwnProperty(d)&&(r.theme=p(t.themes[d],null)),u=!0;else if(A.flags.font.match(d))r.font=A.flags.font.output(d),u=!0;else if(A.flags.auto.match(d))r.auto=!0,u=!0;else if(A.flags.text.match(d))r.text=A.flags.text.output(d),u=!0;else if(A.flags.random.match(d)){null==A.vars.cache.themeKeys&&(A.vars.cache.themeKeys=Object.keys(t.themes));var f=A.vars.cache.themeKeys[0|Math.random()*A.vars.cache.themeKeys.length];r.theme=p(t.themes[f],null),u=!0}u&&r.holderURL.push(d)}return r.holderURL.unshift(t.domain),r.holderURL=r.holderURL.join("/"),n?r:!1}function i(e,t,r,n){var i=r.dimensions,a=r.theme,l=i.width+"x"+i.height;if(e=null==e?r.fluid?"fluid":"image":e,null!=r.text&&(a.text=r.text,"object"===t.nodeName.toLowerCase())){for(var d=a.text.split("\\n"),u=0;u16?a.text.substring(0,16)+"…":a.text)+" ["+l+"]":l}),"image"==e?("html"!=g.renderer&&r.auto||(t.style.width=i.width+"px",t.style.height=i.height+"px"),"html"==g.renderer?t.style.backgroundColor=a.background:(o(e,{dimensions:i,theme:a,flags:r},t,g),r.textmode&&"exact"==r.textmode&&(A.vars.resizableImages.push(t),s(t)))):"background"==e&&"html"!=g.renderer?o(e,{dimensions:i,theme:a,flags:r},t,g):"fluid"==e&&("%"==i.height.slice(-1)?t.style.height=i.height:null!=r.auto&&r.auto||(t.style.height=i.height+"px"),"%"==i.width.slice(-1)?t.style.width=i.width:null!=r.auto&&r.auto||(t.style.width=i.width+"px"),("inline"==t.style.display||""===t.style.display||"none"==t.style.display)&&(t.style.display="block"),h(t),"html"==g.renderer?t.style.backgroundColor=a.background:(A.vars.resizableImages.push(t),s(t)))}function o(e,t,r,n){function i(){var e=null;switch(n.renderer){case"canvas":e=L(s);break;case"svg":e=O(s,n);break;default:throw"Holder: invalid renderer: "+n.renderer}return e}var o=null;switch(n.renderer){case"svg":if(!A.setup.supportsSVG)return;break;case"canvas":if(!A.setup.supportsCanvas)return;break;default:return}{var l={width:t.dimensions.width,height:t.dimensions.height,theme:t.theme,flags:t.flags},s=a(l);({text:l.text,width:l.width,height:l.height,textHeight:l.font.size,font:l.font.family,fontWeight:l.font.weight,template:l.theme})}if(o=i(),null==o)throw"Holder: couldn't render placeholder";"background"==e?(r.style.backgroundImage="url("+o+")",r.style.backgroundSize=l.width+"px "+l.height+"px"):("img"===r.nodeName.toLowerCase()?c(r,{src:o}):"object"===r.nodeName.toLowerCase()&&(c(r,{data:o}),c(r,{type:"image/svg+xml"})),n.reRender&&setTimeout(function(){var e=i();if(null==e)throw"Holder: couldn't render placeholder";"img"===r.nodeName.toLowerCase()?c(r,{src:e}):"object"===r.nodeName.toLowerCase()&&(c(r,{data:e}),c(r,{type:"image/svg+xml"}))},100)),c(r,{"data-holder-rendered":!0})}function a(e){function t(e,t,r,n){t.width=r,t.height=n,e.width=Math.max(e.width,t.width),e.height+=t.height,e.add(t)}switch(e.font={family:e.theme.font?e.theme.font:"Arial, Helvetica, Open Sans, sans-serif",size:l(e.width,e.height,e.theme.size?e.theme.size:A.defaults.size),units:e.theme.units?e.theme.units:A.defaults.units,weight:e.theme.fontweight?e.theme.fontweight:"bold"},e.text=e.theme.text?e.theme.text:Math.floor(e.width)+"x"+Math.floor(e.height),e.flags.textmode){case"literal":e.text=e.flags.dimensions.width+"x"+e.flags.dimensions.height;break;case"exact":if(!e.flags.exactDimensions)break;e.text=Math.floor(e.flags.exactDimensions.width)+"x"+Math.floor(e.flags.exactDimensions.height)}var r=new z({width:e.width,height:e.height}),n=r.Shape,i=new n.Rect("holderBg",{fill:e.theme.background});i.resize(e.width,e.height),r.root.add(i);var o=new n.Group("holderTextGroup",{text:e.text,align:"center",font:e.font,fill:e.theme.foreground});o.moveTo(null,null,1),r.root.add(o);var a=o.textPositionData=T(r);if(!a)throw"Holder: staging fallback not supported yet.";o.properties.leading=a.boundingBox.height;var s=null,d=null;if(a.lineCount>1){var h=0,u=0,c=e.width*A.setup.lineWrapRatio,f=0;d=new n.Group("line"+f);for(var g=0;g=c||m===!0)&&(t(o,d,h,o.properties.leading),h=0,u+=o.properties.leading,f+=1,d=new n.Group("line"+f),d.y=u),m!==!0&&(s.moveTo(h,0),h+=a.spaceWidth+p.width,d.add(s))}t(o,d,h,o.properties.leading);for(var v in o.children)d=o.children[v],d.moveTo((o.width-d.width)/2,null,null);o.moveTo((e.width-o.width)/2,(e.height-o.height)/2,null),(e.height-o.height)/2<0&&o.moveTo(null,0,null)}else s=new n.Text(e.text),d=new n.Group("line0"),d.add(s),o.add(d),o.moveTo((e.width-a.boundingBox.width)/2,(e.height-a.boundingBox.height)/2,null);return r}function l(e,t,r){t=parseInt(t,10),e=parseInt(e,10);var n=Math.max(t,e),i=Math.min(t,e),o=A.defaults.scale,a=Math.min(.75*i,.75*n*o);return Math.round(Math.max(r,a))}function s(e){var t;t=null==e||null==e.nodeType?A.vars.resizableImages:[e];for(var r in t)if(t.hasOwnProperty(r)){var n=t[r];if(n.holderData){var i=n.holderData.flags,a=d(n,k.invisibleErrorFn(s));if(a){if(i.fluid&&i.auto){var l=n.holderData.fluidConfig;switch(l.mode){case"width":a.height=a.width/l.ratio;break;case"height":a.width=a.height*l.ratio}}var h={dimensions:a,theme:i.theme,flags:i};i.textmode&&"exact"==i.textmode&&(i.exactDimensions=a,h.dimensions=i.dimensions),o("image",h,n,n.holderData.renderSettings)}}}}function d(e,t){var r={height:e.clientHeight,width:e.clientWidth};return r.height||r.width?(e.removeAttribute("data-holder-invisible"),r):(c(e,{"data-holder-invisible":!0}),t.call(this,e),void 0)}function h(e){if(e.holderData){var t=d(e,k.invisibleErrorFn(h));if(t){var r=e.holderData.flags,n={fluidHeight:"%"==r.dimensions.height.slice(-1),fluidWidth:"%"==r.dimensions.width.slice(-1),mode:null,initialDimensions:t};n.fluidWidth&&!n.fluidHeight?(n.mode="width",n.ratio=n.initialDimensions.width/parseFloat(r.dimensions.height)):!n.fluidWidth&&n.fluidHeight&&(n.mode="height",n.ratio=parseFloat(r.dimensions.width)/n.initialDimensions.height),e.holderData.fluidConfig=n}}}function u(e,t){return null==t?E.createElement(e):E.createElementNS(t,e)}function c(e,t){for(var r in t)e.setAttribute(r,t[r])}function f(e,t,r){if(null==e){e=u("svg",C);var n=u("defs",C);e.appendChild(n)}return e.webkitMatchesSelector&&e.setAttribute("xmlns",C),c(e,{width:t,height:r,viewBox:"0 0 "+t+" "+r,preserveAspectRatio:"none"}),e}function g(e,r){if(t.XMLSerializer){{var n=new XMLSerializer,i="",o=r.stylesheets;e.querySelector("defs")}if(r.svgXMLStylesheet){for(var a=(new DOMParser).parseFromString("","application/xml"),l=o.length-1;l>=0;l--){var s=a.createProcessingInstruction("xml-stylesheet",'href="'+o[l]+'" rel="stylesheet"');a.insertBefore(s,a.firstChild)}var d=a.createProcessingInstruction("xml",'version="1.0" encoding="UTF-8" standalone="yes"');a.insertBefore(d,a.firstChild),a.removeChild(a.documentElement),i=n.serializeToString(a)}var h=n.serializeToString(e);return h=h.replace(/\&(\#[0-9]{2,}\;)/g,"&$1"),i+h}}function p(e,t){var r={};for(var n in e)e.hasOwnProperty(n)&&(r[n]=e[n]);if(null!=t)for(var i in t)t.hasOwnProperty(i)&&(r[i]=t[i]);return r}function m(e){var t=[];for(var r in e)e.hasOwnProperty(r)&&t.push(r+":"+e[r]);return t.join(";")}function v(e){A.vars.debounceTimer||e.call(this),A.vars.debounceTimer&&clearTimeout(A.vars.debounceTimer),A.vars.debounceTimer=setTimeout(function(){A.vars.debounceTimer=null,e.call(this)},A.setup.debounce)}function x(){v(function(){s(null)})}function y(e){var r=null;return"string"==typeof e?r=E.querySelectorAll(e):t.NodeList&&e instanceof t.NodeList?r=e:t.Node&&e instanceof t.Node?r=[e]:t.HTMLCollection&&e instanceof t.HTMLCollection?r=e:null===e&&(r=[]),r}function w(e,t){var r=new Image;r.onerror=function(){t.call(this,!1)},r.onload=function(){t.call(this,!0)},r.src=e}function b(e){for(var t=[],r=0,n=e.length-1;n>=0;n--)r=e.charCodeAt(n),r>128?t.unshift(["&#",r,";"].join("")):t.unshift(e[n]);return t.join("")}function S(e){return e.replace(/&#(\d+);/g,function(e,t){return String.fromCharCode(t)})}var C="http://www.w3.org/2000/svg",E=t.document,k={addTheme:function(e,t){return null!=e&&null!=t&&(A.settings.themes[e]=t),delete A.vars.cache.themeKeys,this},addImage:function(e,t){var r=E.querySelectorAll(t);if(r.length)for(var n=0,i=r.length;i>n;n++){var o=u("img");c(o,{"data-src":e}),r[n].appendChild(o)}return this},run:function(e){e=e||{};var o={};A.vars.preempted=!0;var a=p(A.settings,e);o.renderer=a.renderer?a.renderer:A.setup.renderer,-1===A.setup.renderers.join(",").indexOf(o.renderer)&&(o.renderer=A.setup.supportsSVG?"svg":A.setup.supportsCanvas?"canvas":"html"),a.use_canvas?o.renderer="canvas":a.use_svg&&(o.renderer="svg");var l=y(a.images),s=y(a.bgnodes),d=y(a.stylenodes),h=y(a.objects);o.stylesheets=[],o.svgXMLStylesheet=!0,o.noFontFallback=a.noFontFallback?a.noFontFallback:!1;for(var c=0;c1){r.nodeValue="";for(var b=0;b -{% block jslibraries %} - - -{% endblock jslibraries %} - -{% block customjs %} - -{% endblock customjs %} - -{% block css_includes %} - -{% endblock css_includes %} - -{% block custom_css %} - -{% endblock custom_css %} - -{% block misc_head_tags %} - SampleClean Dashboard - -{% endblock misc_head_tags %} + + + + + + + + + + Dashboard Template for Bootstrap + + + + + + + + + - -
-
- {% block page_header %} - - {% endblock page_header %} -
-
- {% block main_panel %} -
-
- Registered Queries: - -
-
-
-
-
+
+ + + +
+
+
+ + + +
+
+

Dashboard

+ +
+
+ Generic placeholder thumbnail +

Label

+ Something else +
+
+ Generic placeholder thumbnail +

Label

+ Something else
- {% endblock main_panel %} +
+ Generic placeholder thumbnail +

Label

+ Something else +
+
+ Generic placeholder thumbnail +

Label

+ Something else +
+
+ +

Section title

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#HeaderHeaderHeaderHeader
1,001Loremipsumdolorsit
1,002ametconsecteturadipiscingelit
1,003IntegernecodioPraesent
1,003liberoSedcursusante
1,004dapibusdiamSednisi
1,005Nullaquissemat
1,006nibhelementumimperdietDuis
1,007sagittisipsumPraesentmauris
1,008Fuscenectellussed
1,009auguesemperportaMauris
1,010massaVestibulumlaciniaarcu
1,011egetnullaClassaptent
1,012tacitisociosquadlitora
1,013torquentperconubianostra
1,014perinceptoshimenaeosCurabitur
1,015sodalesligulainlibero
+
+
+
-
+ + + + + + - + \ No newline at end of file From e8552864af9b5da9b4a9125657176559ad209cec Mon Sep 17 00:00:00 2001 From: Pedro Rodriguez Date: Tue, 28 Apr 2015 18:50:42 -0700 Subject: [PATCH 06/12] did some more work on reactjs dashboard --- .../templates/results_dashboard/index.html | 335 +++++++----------- 1 file changed, 128 insertions(+), 207 deletions(-) diff --git a/ampcrowd/results_dashboard/templates/results_dashboard/index.html b/ampcrowd/results_dashboard/templates/results_dashboard/index.html index 2d6a9be..13c687c 100644 --- a/ampcrowd/results_dashboard/templates/results_dashboard/index.html +++ b/ampcrowd/results_dashboard/templates/results_dashboard/index.html @@ -3,7 +3,7 @@ - + @@ -21,228 +21,149 @@ -
- - -
-
+
+
- - - + + +
-

Dashboard

+

Dashboard

-
-
- Generic placeholder thumbnail -

Label

- Something else -
-
- Generic placeholder thumbnail -

Label

- Something else -
-
- Generic placeholder thumbnail -

Label

- Something else +
+
+ Generic placeholder thumbnail +

Label

+ Something else +
+
+ Generic placeholder thumbnail +

Label

+ Something else +
+
+ Generic placeholder thumbnail +

Label

+ Something else +
+
+ Generic placeholder thumbnail +

Label

+ Something else +
-
- Generic placeholder thumbnail -

Label

- Something else -
-
-

Section title

-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#HeaderHeaderHeaderHeader
1,001Loremipsumdolorsit
1,002ametconsecteturadipiscingelit
1,003IntegernecodioPraesent
1,003liberoSedcursusante
1,004dapibusdiamSednisi
1,005Nullaquissemat
1,006nibhelementumimperdietDuis
1,007sagittisipsumPraesentmauris
1,008Fuscenectellussed
1,009auguesemperportaMauris
1,010massaVestibulumlaciniaarcu
1,011egetnullaClassaptent
1,012tacitisociosquadlitora
1,013torquentperconubianostra
1,014perinceptoshimenaeosCurabitur
1,015sodalesligulainlibero
-
+

Crowd Task Groups

+ +
+
-
+
- - - - - + + + + + \ No newline at end of file From 693cff31df86eac11119098cf97afdfb15b141c3 Mon Sep 17 00:00:00 2001 From: Pedro Rodriguez Date: Wed, 29 Apr 2015 10:03:08 -0700 Subject: [PATCH 07/12] ignore "wreckless" changes in this commit, trying to get a working demo, will restore afterwards" --- ampcrowd/basecrowd/models.py | 20 +++++++ ampcrowd/basecrowd/urls.py | 2 + ampcrowd/basecrowd/views.py | 16 +++++ ampcrowd/internal/interface.py | 10 +++- .../templates/results_dashboard/index.html | 58 ++++++++++--------- requirements.txt | 1 + scripts/post.py | 10 ++-- 7 files changed, 85 insertions(+), 32 deletions(-) diff --git a/ampcrowd/basecrowd/models.py b/ampcrowd/basecrowd/models.py index 376b0df..c8427e6 100644 --- a/ampcrowd/basecrowd/models.py +++ b/ampcrowd/basecrowd/models.py @@ -28,6 +28,13 @@ def __unicode__(self): class Meta: abstract = True + @staticmethod + def to_json(task_group): + return { + 'group_id': task_group.group_id, + 'tasks_finished': task_group.tasks_finished, + } + # Model for an individual task. class AbstractCrowdTask(models.Model): @@ -68,6 +75,19 @@ def __unicode__(self): class Meta: abstract = True + @staticmethod + def to_json(task): + return { + 'task_type': task.task_type, + 'data': task.data, + 'creation_time': task.create_time, + 'task_id': task.task_id, + 'num_assignments': task.num_assignments, + 'mv_answer': task.mv_answer, + 'em_answer': task.em_answer, + 'is_complete': task.is_complete + } + # Model for workers class AbstractCrowdWorker(models.Model): diff --git a/ampcrowd/basecrowd/urls.py b/ampcrowd/basecrowd/urls.py index e752862..45f8cc3 100644 --- a/ampcrowd/basecrowd/urls.py +++ b/ampcrowd/basecrowd/urls.py @@ -8,4 +8,6 @@ url(r'^(\w+)/responses/$', views.post_response, name='post_response'), url(r'^(\w+)/tasks/$', views.create_task_group, name='create_tasks'), url(r'^(\w+)/purge_tasks/$', views.purge_tasks, name='purge_tasks'), + url(r'^(\w+)/summary/task_groups', views.get_task_groups, name='get_task_groups'), + url(r'^(\w+)/summary/tasks', views.get_tasks, name="get_tasks") ) diff --git a/ampcrowd/basecrowd/views.py b/ampcrowd/basecrowd/views.py index 3d43e55..d004652 100644 --- a/ampcrowd/basecrowd/views.py +++ b/ampcrowd/basecrowd/views.py @@ -5,6 +5,7 @@ from django.views.decorators.http import require_GET, require_POST from django.http import HttpResponse from datetime import datetime +from jsonview.decorators import json_view import pytz import json import os @@ -16,6 +17,21 @@ logger = logging.getLogger('crowd_server') +@require_GET +@json_view +def get_task_groups(request, crowd_name): + interface, model_spec = CrowdRegistry.get_registry_entry(crowd_name) + logger.debug(type(interface)) + logger.debug(type(model_spec)) + return interface.get_task_groups() + +@require_GET +@json_view +def get_tasks(request, crowd_name): + interface, model_spec = CrowdRegistry.get_registry_entry(crowd_name) + return interface.get_tasks() + + @require_POST @csrf_exempt def create_task_group(request, crowd_name): diff --git a/ampcrowd/internal/interface.py b/ampcrowd/internal/interface.py index 2ce6ff3..cc7654f 100644 --- a/ampcrowd/internal/interface.py +++ b/ampcrowd/internal/interface.py @@ -5,7 +5,7 @@ from django.db.models import Count, F, Q from basecrowd.interface import CrowdInterface -from models import CrowdTask +from models import CrowdTask, CrowdTaskGroup SLACK = 2 ORDER_WEIGHT = 0.3 @@ -64,4 +64,12 @@ def get_eligible_tasks(worker_id): # No duplicates .distinct()) + @staticmethod + def get_task_groups(): + return CrowdTaskGroup.objects.all() + + @staticmethod + def get_tasks(): + return map(CrowdTask.to_json, CrowdTask.objects.all()) + INTERNAL_CROWD_INTERFACE = InternalCrowdInterface('internal') diff --git a/ampcrowd/results_dashboard/templates/results_dashboard/index.html b/ampcrowd/results_dashboard/templates/results_dashboard/index.html index 13c687c..e3f5a64 100644 --- a/ampcrowd/results_dashboard/templates/results_dashboard/index.html +++ b/ampcrowd/results_dashboard/templates/results_dashboard/index.html @@ -101,29 +101,33 @@

Label

Crowd Task Groups

diff --git a/requirements.txt b/requirements.txt index cc0021b..de24f0f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ billiard==3.3.0.18 boto==2.29.1 celery==3.1.15 django-celery==3.1.16 +django-jsonview==0.4.3 django-sslserver==0.12 gunicorn==19.0.0 kombu==3.0.23 diff --git a/scripts/post.py b/scripts/post.py index 9041774..0c2dc7a 100755 --- a/scripts/post.py +++ b/scripts/post.py @@ -37,7 +37,7 @@ def send_request(data, crowds, num_requests, use_ssl): # Send request params = {'data' : json.dumps(data)} scheme = 'https' if use_ssl else 'http' - url = scheme + '://127.0.0.1:8000/crowds/%s/tasks/' + url = scheme + '://datascijenkins-0-1.sv2.trulia.com:4040/crowds/%s/tasks/' for crowd in crowds: for i in range(num_requests): try: @@ -105,11 +105,13 @@ def create_tasks(crowds, task_types, use_ssl): data['configuration']['amt'] = {'sandbox' : True} data['group_id'] = 'test2' - data['group_context'] = {'fields' : ['price', 'location']} + data['group_context'] = {'fields' : ['company', 'location']} # This configuration generates one task with two pairs of records. - data['content'] = {'pair1' : [['5', 'LA'], ['6', 'Berkeley']], - 'pair2' : [['80', 'London'], ['80.0', 'Londyn']]} + data['content'] = {'pair1' : [['Zillow Group', 'Seattle'], ['Zillow Group', 'San Francisco']], + 'pair2' : [['Trulia', 'San Francisco'], ['Trulia', 'New York']], + 'pair3' : [['Trulia', 'San Francisco'], ['Hotpads', 'San Francisco']], + 'pair4' : [['Zillow', 'Seattle'], ['Trulia', 'San Francisco']]} send_request(data, crowds, num_tasks, use_ssl) print "Done!" From 12ad933e7adfd0881ed3a0887f73138f0e4657b0 Mon Sep 17 00:00:00 2001 From: Pedro Rodriguez Date: Wed, 29 Apr 2015 10:16:51 -0700 Subject: [PATCH 08/12] better tracking --- ampcrowd/basecrowd/models.py | 3 ++- .../results_dashboard/templates/results_dashboard/index.html | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ampcrowd/basecrowd/models.py b/ampcrowd/basecrowd/models.py index c8427e6..6a9c44a 100644 --- a/ampcrowd/basecrowd/models.py +++ b/ampcrowd/basecrowd/models.py @@ -85,7 +85,8 @@ def to_json(task): 'num_assignments': task.num_assignments, 'mv_answer': task.mv_answer, 'em_answer': task.em_answer, - 'is_complete': task.is_complete + 'is_complete': task.is_complete, + 'completed_assignments': len(task.responses.all()) } diff --git a/ampcrowd/results_dashboard/templates/results_dashboard/index.html b/ampcrowd/results_dashboard/templates/results_dashboard/index.html index e3f5a64..f5de869 100644 --- a/ampcrowd/results_dashboard/templates/results_dashboard/index.html +++ b/ampcrowd/results_dashboard/templates/results_dashboard/index.html @@ -124,6 +124,7 @@

Crowd Task Groups

{row.task_id} {row.data} {row.is_complete} + {row.completed_assignments} {row.num_assignments} {row.mv_answer} {row.em_answer} @@ -138,7 +139,8 @@

Crowd Task Groups

Task ID Data Is Complete - Assignments + Completed Assignments + Total Assignments Majority Vote Answer EM Answer Task Type From dc08649ca3ad0f40fc9132cd41888886a1f6e5a8 Mon Sep 17 00:00:00 2001 From: Pedro Rodriguez Date: Wed, 29 Apr 2015 16:47:50 -0700 Subject: [PATCH 09/12] trimming down requirements.txt --- Dockerfile | 1 + ampcrowd/internal/views.py | 2 +- ampcrowd/results_dashboard/views.py | 1 - requirements.txt | 8 +------- scripts/post.py | 2 +- 5 files changed, 4 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index b3c9ddc..902cac7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,7 @@ RUN apt-get update && apt-get install -y postgresql-client COPY requirements.txt /usr/src/app/ WORKDIR /usr/src/app +RUN pip install numpy RUN pip install -r requirements.txt ENV PYTHONUNBUFFERED 1 diff --git a/ampcrowd/internal/views.py b/ampcrowd/internal/views.py index ad1ff31..2371ffc 100644 --- a/ampcrowd/internal/views.py +++ b/ampcrowd/internal/views.py @@ -50,7 +50,7 @@ def index(request): .values('task_type') .annotate(num_tasks=Count('task_id'))) - task_types = { t['task_type'] : + task_types = {t['task_type']: build_context(task_type_map, total_tasks_by_type, worker_id, task_type_obj=t) for t in incomplete_tasks_by_type } diff --git a/ampcrowd/results_dashboard/views.py b/ampcrowd/results_dashboard/views.py index 8f995ca..2d7bdf1 100644 --- a/ampcrowd/results_dashboard/views.py +++ b/ampcrowd/results_dashboard/views.py @@ -1,6 +1,5 @@ import json import pytz -import uuid from datetime import datetime from django.http import HttpResponse from django.shortcuts import render diff --git a/requirements.txt b/requirements.txt index de24f0f..0c8179e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,11 @@ Django==1.7 amqp==1.4.6 -anyjson==0.3.3 -billiard==3.3.0.18 boto==2.29.1 celery==3.1.15 django-celery==3.1.16 django-jsonview==0.4.3 django-sslserver==0.12 gunicorn==19.0.0 -kombu==3.0.23 -matplotlib==1.4.0 -mechanize==0.2.5 -mock==1.0.1 -multi-mechanize==1.2.0 nose==1.3.4 numpy==1.9.0 psycopg2==2.5.3 @@ -21,3 +14,4 @@ python-dateutil==2.2 pytz==2014.7 six==1.8.0 wsgiref==0.1.2 + diff --git a/scripts/post.py b/scripts/post.py index 0c2dc7a..86a835c 100755 --- a/scripts/post.py +++ b/scripts/post.py @@ -37,7 +37,7 @@ def send_request(data, crowds, num_requests, use_ssl): # Send request params = {'data' : json.dumps(data)} scheme = 'https' if use_ssl else 'http' - url = scheme + '://datascijenkins-0-1.sv2.trulia.com:4040/crowds/%s/tasks/' + url = scheme + '://crowd_server:8000/crowds/%s/tasks/' for crowd in crowds: for i in range(num_requests): try: From 97f0259d03b1f61dcbac409d2af115f9d87dcf3f Mon Sep 17 00:00:00 2001 From: Pedro Rodriguez Date: Wed, 29 Apr 2015 16:52:20 -0700 Subject: [PATCH 10/12] removed very samll numpy dependency. Adds a lot of overhead on builds when ampcrowd doesn't really use numpy except for some random number skewing. --- ampcrowd/internal/interface.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ampcrowd/internal/interface.py b/ampcrowd/internal/interface.py index cc7654f..9b46c4e 100644 --- a/ampcrowd/internal/interface.py +++ b/ampcrowd/internal/interface.py @@ -1,5 +1,5 @@ import uuid -from numpy.random import geometric +from random import randint from django.core.urlresolvers import reverse from django.db.models import Count, F, Q @@ -23,8 +23,7 @@ def get_assignment_context(request): .order_by('create_time')) # Pick a random task, biased towards older tasks. - task_index = min(geometric(ORDER_WEIGHT) - 1, - eligible_tasks.count() - 1) + task_index = randint(0, eligible_tasks.count() - 1) # generate a random assignment id for this assignment. assignment_id = uuid.uuid4() From bd398c154a0df58152bddaf4f714c8e2c2f5271a Mon Sep 17 00:00:00 2001 From: Pedro Rodriguez Date: Thu, 30 Apr 2015 20:43:13 -0700 Subject: [PATCH 11/12] reversions and light refactor --- Dockerfile | 1 - ampcrowd/basecrowd/models.py | 2 +- ampcrowd/internal/interface.py | 2 +- requirements.txt | 1 - scripts/post.py | 8 ++++---- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 902cac7..b3c9ddc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,6 @@ RUN apt-get update && apt-get install -y postgresql-client COPY requirements.txt /usr/src/app/ WORKDIR /usr/src/app -RUN pip install numpy RUN pip install -r requirements.txt ENV PYTHONUNBUFFERED 1 diff --git a/ampcrowd/basecrowd/models.py b/ampcrowd/basecrowd/models.py index 6a9c44a..fa1ee7f 100644 --- a/ampcrowd/basecrowd/models.py +++ b/ampcrowd/basecrowd/models.py @@ -76,7 +76,7 @@ class Meta: abstract = True @staticmethod - def to_json(task): + def to_dict(task): return { 'task_type': task.task_type, 'data': task.data, diff --git a/ampcrowd/internal/interface.py b/ampcrowd/internal/interface.py index 9b46c4e..f74e93f 100644 --- a/ampcrowd/internal/interface.py +++ b/ampcrowd/internal/interface.py @@ -69,6 +69,6 @@ def get_task_groups(): @staticmethod def get_tasks(): - return map(CrowdTask.to_json, CrowdTask.objects.all()) + return map(CrowdTask.to_dict, CrowdTask.objects.all()) INTERNAL_CROWD_INTERFACE = InternalCrowdInterface('internal') diff --git a/requirements.txt b/requirements.txt index 0c8179e..f331425 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,6 @@ django-jsonview==0.4.3 django-sslserver==0.12 gunicorn==19.0.0 nose==1.3.4 -numpy==1.9.0 psycopg2==2.5.3 pyparsing==2.0.2 python-dateutil==2.2 diff --git a/scripts/post.py b/scripts/post.py index 86a835c..e2d72ef 100755 --- a/scripts/post.py +++ b/scripts/post.py @@ -108,10 +108,10 @@ def create_tasks(crowds, task_types, use_ssl): data['group_context'] = {'fields' : ['company', 'location']} # This configuration generates one task with two pairs of records. - data['content'] = {'pair1' : [['Zillow Group', 'Seattle'], ['Zillow Group', 'San Francisco']], - 'pair2' : [['Trulia', 'San Francisco'], ['Trulia', 'New York']], - 'pair3' : [['Trulia', 'San Francisco'], ['Hotpads', 'San Francisco']], - 'pair4' : [['Zillow', 'Seattle'], ['Trulia', 'San Francisco']]} + data['content'] = { + 'pair1' : [['5', 'LA'], ['6', 'Berkeley']], +- 'pair2' : [['80', 'London'], ['80.0', 'Londyn']] + } send_request(data, crowds, num_tasks, use_ssl) print "Done!" From 3da6d640dc1a7300436dc78816860e32cb5bb1d7 Mon Sep 17 00:00:00 2001 From: Pedro Rodriguez Date: Fri, 1 May 2015 00:36:08 -0700 Subject: [PATCH 12/12] refactored where models are, renamed task fields for clarity --- ampcrowd/amt/models.py | 6 +- ampcrowd/basecrowd/abstract_crowd_models.py | 165 +++++++++++++++ ampcrowd/basecrowd/interface.py | 2 +- ampcrowd/basecrowd/models.py | 215 +------------------- ampcrowd/basecrowd/task_models.py | 40 ++++ ampcrowd/basecrowd/tests/test_models.py | 28 +-- ampcrowd/basecrowd/views.py | 3 +- ampcrowd/docker-entrypoint.sh | 2 +- ampcrowd/internal/models.py | 7 +- 9 files changed, 228 insertions(+), 240 deletions(-) create mode 100644 ampcrowd/basecrowd/abstract_crowd_models.py create mode 100644 ampcrowd/basecrowd/task_models.py diff --git a/ampcrowd/amt/models.py b/ampcrowd/amt/models.py index 9eeaa9b..4312d57 100644 --- a/ampcrowd/amt/models.py +++ b/ampcrowd/amt/models.py @@ -1,8 +1,6 @@ from django.db import models -from basecrowd.models import AbstractCrowdTaskGroup -from basecrowd.models import AbstractCrowdTask -from basecrowd.models import AbstractCrowdWorker -from basecrowd.models import AbstractCrowdWorkerResponse +from basecrowd.abstract_crowd_models import AbstractCrowdTask, AbstractCrowdWorker, AbstractCrowdWorkerResponse, \ + AbstractCrowdTaskGroup # Inherited crowd models for the interface. # No need for special subclasses, we use the base implementations. diff --git a/ampcrowd/basecrowd/abstract_crowd_models.py b/ampcrowd/basecrowd/abstract_crowd_models.py new file mode 100644 index 0000000..6e23783 --- /dev/null +++ b/ampcrowd/basecrowd/abstract_crowd_models.py @@ -0,0 +1,165 @@ +from django.db import models + + +class AbstractCrowdTask(models.Model): + + # The group that this task belongs to, a many-to-one relationship. + # The relationship will be auto-generated to the task_group class of the + # registered crowd, and can be accessed via the 'group' attribute. + # The related_name will be 'tasks' to enable reverse lookups, e.g. + # group = models.ForeignKey(CrowdTaskGroup, related_name='tasks') + + # The type of the task, Sentiment Analysis, Deduplication, etc + task_type = models.CharField(max_length=64) + + # The data for the task, specific to the task type (stored as a JSON blob) + data = models.TextField() + + # Creation time + create_time = models.DateTimeField() + + # Unique identifier for the task + task_id = models.CharField(primary_key=True, max_length=64) + + # The number of assignments (i.e., number of votes) to get for this task. + num_assignments = models.IntegerField() + + # Answer based on majority vote + mv_answer = models.TextField() + + # Answer based on Expectation Maximization + em_answer = models.TextField() + + # Has the task received enough responses? + is_complete = models.BooleanField(default=False) + + def __unicode__(self): + return self.task_type + " : " + self.data + + class Meta: + abstract = True + + @staticmethod + def to_dict(task): + return { + 'task_type': task.task_type, + 'data': task.data, + 'creation_time': task.create_time, + 'task_id': task.task_id, + 'num_assignments': task.num_assignments, + 'mv_answer': task.mv_answer, + 'em_answer': task.em_answer, + 'is_complete': task.is_complete, + 'completed_assignments': len(task.responses.all()) + } + + +class AbstractCrowdWorker(models.Model): + + # The tasks a worker has been assigned, a many-to-many relationship. + # The relationship will be auto-generated to the task class of the + # registered crowd, and can be accessed via the 'tasks' attribute. + # The related_name will be 'workers' to enable reverse lookups, e.g. + # tasks = models.ManyToManyField(CrowdTask, related_name='workers') + + # A unique id for the worker + worker_id = models.CharField(primary_key=True, max_length=64) + + def __unicode__(self): + return self.worker_id + + class Meta: + abstract = True + + +class AbstractCrowdWorkerResponse(models.Model): + + # The task that was responded to, a many-to-one relationship. + # The relationship will be auto-generated to the task class of the + # registered crowd, and can be accessed via the 'task' attribute. + # The related_name will be 'responses' to enable reverse lookups, e.g. + # task = models.ForeignKey(CrowdTask, related_name='responses') + + # The worker that gave the responded, a many-to-one relationship. + # The relationship will be auto-generated to the worker class of the + # registered crowd, and can be accessed via the 'worker' attribute. + # The related_name will be 'responses' to enable reverse lookups, e.g. + # worker = models.ForeignKey(CrowdWorker, related_name='responses') + + # The content of the response (specific to the task type). + content = models.TextField() + + # The assignment id of this response + assignment_id = models.CharField(max_length=200) + + def __unicode__(self): + return self.task.task_id + " " + self.worker.worker_id + + class Meta: + abstract = True + + +class AbstractCrowdTaskGroup(models.Model): + + # The group id + group_id = models.CharField(primary_key=True, max_length=64) + + # The number of HITs in this group that have been finished + tasks_finished = models.IntegerField() + + # The call back URL for sending results once complete + callback_url = models.URLField() + + # Context for rendering the tasks to the crowd, as a JSON blob. + group_context = models.TextField() + + # The configuration specific to current crowd type + crowd_config = models.TextField() + + def __unicode__(self): + return self.group_id + + class Meta: + abstract = True + + @staticmethod + def to_dict(task_group): + return { + 'group_id': task_group.group_id, + 'tasks_finished': task_group.tasks_finished, + } + + +class CrowdModelSpecification(object): + def __init__(self, crowd_name, + task_model, + group_model, + worker_model, + response_model): + self.name = crowd_name + self.task_model = task_model + self.group_model = group_model + self.worker_model = worker_model + self.response_model = response_model + + @staticmethod + def add_rel(from_cls, to_cls, relation_cls, relation_name, related_name=None): + field = relation_cls(to_cls, related_name=related_name) + field.contribute_to_class(from_cls, relation_name) + + def add_model_rels(self): + # tasks belong to groups + self.add_rel(self.task_model, self.group_model, models.ForeignKey, + 'group', 'tasks') + + # workers work on many tasks, each task might have multiple workers + self.add_rel(self.worker_model, self.task_model, models.ManyToManyField, + 'tasks', 'workers') + + # responses come from a worker + self.add_rel(self.response_model, self.worker_model, models.ForeignKey, + 'worker', 'responses') + + # responses pertain to a task + self.add_rel(self.response_model, self.task_model, models.ForeignKey, + 'task', 'responses') \ No newline at end of file diff --git a/ampcrowd/basecrowd/interface.py b/ampcrowd/basecrowd/interface.py index 2e7367d..83c1cbd 100644 --- a/ampcrowd/basecrowd/interface.py +++ b/ampcrowd/basecrowd/interface.py @@ -3,7 +3,7 @@ from django.core.urlresolvers import reverse -from models import CrowdModelSpecification +from basecrowd.abstract_crowd_models import CrowdModelSpecification # Required implementation for a new crowd type diff --git a/ampcrowd/basecrowd/models.py b/ampcrowd/basecrowd/models.py index fa1ee7f..c935039 100644 --- a/ampcrowd/basecrowd/models.py +++ b/ampcrowd/basecrowd/models.py @@ -1,213 +1,2 @@ -from django.db import models -from django.db.models.signals import class_prepared -from django.contrib.contenttypes import generic -from django.contrib.contenttypes.models import ContentType - - -# Model for a group of tasks -class AbstractCrowdTaskGroup(models.Model): - - # The group id - group_id = models.CharField(primary_key=True, max_length=64) - - # The number of HITs in this group that have been finished - tasks_finished = models.IntegerField() - - # The call back URL for sending results once complete - callback_url = models.URLField() - - # Context for rendering the tasks to the crowd, as a JSON blob. - group_context = models.TextField() - - # The configuration specific to current crowd type - crowd_config = models.TextField() - - def __unicode__(self): - return self.group_id - - class Meta: - abstract = True - - @staticmethod - def to_json(task_group): - return { - 'group_id': task_group.group_id, - 'tasks_finished': task_group.tasks_finished, - } - - -# Model for an individual task. -class AbstractCrowdTask(models.Model): - - # The group that this task belongs to, a many-to-one relationship. - # The relationship will be auto-generated to the task_group class of the - # registered crowd, and can be accessed via the 'group' attribute. - # The related_name will be 'tasks' to enable reverse lookups, e.g. - # group = models.ForeignKey(CrowdTaskGroup, related_name='tasks') - - # The type of the task, Sentiment Analysis, Deduplication, etc - task_type = models.CharField(max_length=64) - - # The data for the task, specific to the task type (stored as a JSON blob) - data = models.TextField() - - # Creation time - create_time = models.DateTimeField() - - # Unique identifier for the task - task_id = models.CharField(primary_key=True, max_length=64) - - # The number of assignments (i.e., number of votes) to get for this task. - num_assignments = models.IntegerField() - - # Answer based on majority vote - mv_answer = models.TextField() - - # Answer based on Expectation Maximization - em_answer = models.TextField() - - # Has the task received enough responses? - is_complete = models.BooleanField(default=False) - - def __unicode__(self): - return self.task_type + " : " + self.data - - class Meta: - abstract = True - - @staticmethod - def to_dict(task): - return { - 'task_type': task.task_type, - 'data': task.data, - 'creation_time': task.create_time, - 'task_id': task.task_id, - 'num_assignments': task.num_assignments, - 'mv_answer': task.mv_answer, - 'em_answer': task.em_answer, - 'is_complete': task.is_complete, - 'completed_assignments': len(task.responses.all()) - } - - -# Model for workers -class AbstractCrowdWorker(models.Model): - - # The tasks a worker has been assigned, a many-to-many relationship. - # The relationship will be auto-generated to the task class of the - # registered crowd, and can be accessed via the 'tasks' attribute. - # The related_name will be 'workers' to enable reverse lookups, e.g. - # tasks = models.ManyToManyField(CrowdTask, related_name='workers') - - # A unique id for the worker - worker_id = models.CharField(primary_key=True, max_length=64) - - def __unicode__(self): - return self.worker_id - - class Meta: - abstract = True - - -# Model for a worker's response to a task -class AbstractCrowdWorkerResponse(models.Model): - - # The task that was responded to, a many-to-one relationship. - # The relationship will be auto-generated to the task class of the - # registered crowd, and can be accessed via the 'task' attribute. - # The related_name will be 'responses' to enable reverse lookups, e.g. - # task = models.ForeignKey(CrowdTask, related_name='responses') - - # The worker that gave the responded, a many-to-one relationship. - # The relationship will be auto-generated to the worker class of the - # registered crowd, and can be accessed via the 'worker' attribute. - # The related_name will be 'responses' to enable reverse lookups, e.g. - # worker = models.ForeignKey(CrowdWorker, related_name='responses') - - # The content of the response (specific to the task type). - content = models.TextField() - - # The assignment id of this response - assignment_id = models.CharField(max_length=200) - - def __unicode__(self): - return self.task.task_id + " " + self.worker.worker_id - - class Meta: - abstract = True - - -# Register a set of models as a new crowd. -class CrowdModelSpecification(object): - def __init__(self, crowd_name, - task_model, - group_model, - worker_model, - response_model): - self.name = crowd_name - self.task_model = task_model - self.group_model = group_model - self.worker_model = worker_model - self.response_model = response_model - - @staticmethod - def add_rel(from_cls, to_cls, relation_cls, relation_name, related_name=None): - field = relation_cls(to_cls, related_name=related_name) - field.contribute_to_class(from_cls, relation_name) - - def add_model_rels(self): - # tasks belong to groups - self.add_rel(self.task_model, self.group_model, models.ForeignKey, - 'group', 'tasks') - - # workers work on many tasks, each task might have multiple workers - self.add_rel(self.worker_model, self.task_model, models.ManyToManyField, - 'tasks', 'workers') - - # responses come from a worker - self.add_rel(self.response_model, self.worker_model, models.ForeignKey, - 'worker', 'responses') - - # responses pertain to a task - self.add_rel(self.response_model, self.task_model, models.ForeignKey, - 'task', 'responses') - - -class TemplateResource(models.Model): - name = models.CharField(max_length=200) - content = models.TextField() - direct_dependencies = models.ManyToManyField('self', symmetrical=False, related_name="upstream_dependencies") - direct_requirements = models.ManyToManyField('self', symmetrical=False, related_name="upstream_requirements") - - @property - def dependencies(self): - query_set = list(self.direct_dependencies.all()) - all_dependencies = set(query_set) - current_dependencies = list(query_set) - while True: - prior_size = len(all_dependencies) - new_dependencies = set() - for dep in current_dependencies: - deps = set(dep.direct_dependencies.all()) - new_dependencies = new_dependencies.union(deps) - all_dependencies = all_dependencies.union(deps) - if prior_size == len(all_dependencies): - break - else: - current_dependencies = list(new_dependencies) - return all_dependencies - - @property - def requirements(self): - pass - - - def __str__(self): - return self.name - - -class TaskType(models.Model): - name = models.CharField(max_length=200) - iterator_template = models.ForeignKey(TemplateResource, related_name='iterator_task') - point_template = models.ForeignKey(TemplateResource, related_name='point_task') - renderer = models.ForeignKey(TemplateResource, related_name='renderer_task') +from basecrowd.abstract_crowd_models import * +from basecrowd.task_models import * diff --git a/ampcrowd/basecrowd/task_models.py b/ampcrowd/basecrowd/task_models.py new file mode 100644 index 0000000..7d57cb9 --- /dev/null +++ b/ampcrowd/basecrowd/task_models.py @@ -0,0 +1,40 @@ +from django.db import models + + +class TemplateResource(models.Model): + name = models.CharField(max_length=200) + content = models.TextField() + direct_imports = models.ManyToManyField('self', symmetrical=False, related_name="downstream_imports") + direct_requirements = models.ManyToManyField('self', symmetrical=False, related_name="downstream_requirements") + + @property + def imports(self): + query_set = list(self.direct_imports.all()) + all_imports = set(query_set) + current_imports = list(query_set) + while True: + prior_size = len(all_imports) + new_imports = set() + for dep in current_imports: + deps = set(dep.direct_imports.all()) + new_imports = new_imports.union(deps) + all_imports = all_imports.union(deps) + if prior_size == len(all_imports): + break + else: + current_imports = list(new_imports) + return all_imports + + @property + def requirements(self): + pass + + def __str__(self): + return self.name + + +class TaskType(models.Model): + name = models.CharField(max_length=200) + iterator_template = models.ForeignKey(TemplateResource, related_name='iterator_task') + point_template = models.ForeignKey(TemplateResource, related_name='point_task') + renderer = models.ForeignKey(TemplateResource, related_name='renderer_task') diff --git a/ampcrowd/basecrowd/tests/test_models.py b/ampcrowd/basecrowd/tests/test_models.py index 5256112..6bd5f1f 100644 --- a/ampcrowd/basecrowd/tests/test_models.py +++ b/ampcrowd/basecrowd/tests/test_models.py @@ -1,5 +1,5 @@ from django.test import TestCase -from basecrowd.models import TaskType, TemplateResource +from basecrowd.task_models import TaskType, TemplateResource class TasksTestCase(TestCase): @@ -8,26 +8,26 @@ def setUp(self): js = TemplateResource.objects.create(name="js") css = TemplateResource.objects.create(name="css") bootstrap = TemplateResource.objects.create(name="bootstrap") - bootstrap.direct_dependencies.add(html, js, css) + bootstrap.direct_imports.add(html, js, css) jquery = TemplateResource.objects.create(name="jquery") - jquery.direct_dependencies.add(html, js, css) + jquery.direct_imports.add(html, js, css) reactjs = TemplateResource.objects.create(name="reactjs") - reactjs.direct_dependencies.add(js) + reactjs.direct_imports.add(js) jqueryui = TemplateResource.objects.create(name="jqueryui") - jqueryui.direct_dependencies.add(jquery) + jqueryui.direct_imports.add(jquery) bootstrap_react_gallery = TemplateResource.objects.create(name="bootstrap react gallery") - bootstrap_react_gallery.direct_dependencies.add(reactjs, bootstrap) + bootstrap_react_gallery.direct_imports.add(reactjs, bootstrap) monolithic_app = TemplateResource.objects.create(name="monolithic app") - monolithic_app.direct_dependencies.add( + monolithic_app.direct_imports.add( bootstrap_react_gallery, jqueryui, jquery, js, css, html, reactjs ) - def test_template_dependencies(self): + def test_template_imports(self): html = TemplateResource.objects.get(name="html") css = TemplateResource.objects.get(name="css") js = TemplateResource.objects.get(name="js") @@ -38,26 +38,26 @@ def test_template_dependencies(self): bootstrap_react_gallery = TemplateResource.objects.get(name="bootstrap react gallery") monolithic_app = TemplateResource.objects.get(name="monolithic app") self.assertSetEqual( - jquery.dependencies, + jquery.imports, {html, css, js} ) self.assertSetEqual( - html.dependencies, + html.imports, set() ) self.assertSetEqual( - reactjs.dependencies, + reactjs.imports, {js} ) self.assertSetEqual( - jqueryui.dependencies, + jqueryui.imports, {jquery, html, js, css} ) self.assertSetEqual( - bootstrap_react_gallery.dependencies, + bootstrap_react_gallery.imports, {html, js, css, reactjs, bootstrap} ) self.assertSetEqual( - monolithic_app.dependencies, + monolithic_app.imports, {html, js, css, reactjs, jquery, jqueryui, bootstrap, bootstrap_react_gallery} ) diff --git a/ampcrowd/basecrowd/views.py b/ampcrowd/basecrowd/views.py index d004652..ec0116d 100644 --- a/ampcrowd/basecrowd/views.py +++ b/ampcrowd/basecrowd/views.py @@ -21,10 +21,9 @@ @json_view def get_task_groups(request, crowd_name): interface, model_spec = CrowdRegistry.get_registry_entry(crowd_name) - logger.debug(type(interface)) - logger.debug(type(model_spec)) return interface.get_task_groups() + @require_GET @json_view def get_tasks(request, crowd_name): diff --git a/ampcrowd/docker-entrypoint.sh b/ampcrowd/docker-entrypoint.sh index 9ba09fb..c9dacb6 100644 --- a/ampcrowd/docker-entrypoint.sh +++ b/ampcrowd/docker-entrypoint.sh @@ -2,7 +2,7 @@ # Migrate databases if necessary, sleep allows postgres container to finish launching sleep 3 -python ampcrowd/manage.py syncdb --noinput +python ampcrowd/manage.py migrate --noinput # Generate static content python ampcrowd/manage.py collectstatic --noinput diff --git a/ampcrowd/internal/models.py b/ampcrowd/internal/models.py index f53ecf2..7ff5758 100644 --- a/ampcrowd/internal/models.py +++ b/ampcrowd/internal/models.py @@ -1,8 +1,5 @@ -from django.db import models -from basecrowd.models import AbstractCrowdTaskGroup -from basecrowd.models import AbstractCrowdTask -from basecrowd.models import AbstractCrowdWorker -from basecrowd.models import AbstractCrowdWorkerResponse +from basecrowd.abstract_crowd_models import AbstractCrowdTask, AbstractCrowdWorker, AbstractCrowdWorkerResponse, \ + AbstractCrowdTaskGroup # Inherited crowd models for the interface. # No need for special subclasses, we use the base implementations.