PLAT-1060 Add REST API and other enhancements

Author: jmbowman

.gitignore

@@ -60,6 +60,7 @@ output/*/index.html
 # Sphinx
 docs/_build
 docs/modules.rst
+docs/rest_api.rst
 docs/user_tasks.rst
 
 # Private requirements
@@ -68,3 +69,6 @@ requirements/private.txt
 
 # tox environment temporary artifacts
 tests/__init__.py
+
+# Development task artifacts
+default.db

Makefile

@@ -1,16 +1,25 @@
 .PHONY: clean compile_translations coverage docs dummy_translations extract_translations \
-	fake_translations help pull_translations push_translations quality requirements test test-all validate
+	fake_translations help pull_translations push_translations quality \
+	requirements swagger-ui test test-all upgrade validate
 
 .DEFAULT_GOAL := help
 
 define BROWSER_PYSCRIPT
 import os, webbrowser, sys
+from time import sleep
 try:
 	from urllib import pathname2url
 except:
 	from urllib.request import pathname2url
 
-webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1])))
+path_or_url = sys.argv[1]
+delay = int(sys.argv[2]) if len(sys.argv) > 2 else 0
+if delay > 0:
+    sleep(delay)
+if '://' in path_or_url:
+    webbrowser.open(path_or_url)
+else:
+    webbrowser.open("file://" + pathname2url(os.path.abspath(path_or_url)))
 endef
 export BROWSER_PYSCRIPT
 BROWSER := python -c "$$BROWSER_PYSCRIPT"
@@ -32,7 +41,7 @@ compile_translations: ## compile translation files, outputting .po files for eac
 	./manage.py compilemessages
 
 coverage: clean ## generate and view HTML coverage report
-	py.test --cov-report html
+	pytest --cov-report html
 	$(BROWSER) htmlcov/index.html
 
 docs: ## generate Sphinx HTML documentation, including API docs
@@ -48,11 +57,11 @@ extract_translations: ## extract strings to be translated, outputting .mo files
 
 fake_translations: extract_translations dummy_translations compile_translations ## generate and compile dummy translation files
 
-pip-compile: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
+upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
 	pip install -q pip-tools
 	pip-compile --upgrade -o requirements/base.txt requirements/base.in
-	pip-compile --upgrade -o requirements/dev.txt requirements/dev.in
-	pip-compile --upgrade -o requirements/doc.txt requirements/doc.in
+	pip-compile --upgrade -o requirements/dev.txt requirements/dev.in requirements/quality.in
+	pip-compile --upgrade -o requirements/doc.txt requirements/base.in requirements/doc.in
 	pip-compile --upgrade -o requirements/quality.txt requirements/quality.in
 	pip-compile --upgrade -o requirements/test.txt requirements/base.in requirements/test.in
 	pip-compile --upgrade -o requirements/travis.txt requirements/travis.in
@@ -73,8 +82,14 @@ requirements: ## install development environment requirements
 	pip install -qr requirements/dev.txt --exists-action w
 	pip-sync requirements/base.txt requirements/dev.txt requirements/private.* requirements/test.txt
 
+swagger-ui: ## view Swagger UI for the REST API documentation
+	tox -e docs
+	$(BROWSER) http://localhost:8000 5 &
+	echo "The REST API documentation should open in your browser within a few seconds"
+	. .tox/docs/bin/activate; ./manage.py migrate --settings=schema.settings; SWAGGER_JSON_PATH=docs/swagger.json ./manage.py runserver --settings=schema.settings
+
 test: clean ## run tests in the current virtualenv
-	py.test
+	pytest
 
 test-all: ## run tests on every supported Python/Django combination
 	tox -e quality

docs/authorization.rst

@@ -0,0 +1,50 @@
+Authorization
+=============
+
+Access to task artifacts and status records via the REST API is restricted via Django's authorization mechanism.
+The following permissions are checked when assorted operations are attempted:
+
+* user_tasks.cancel_usertaskstatus
+* user_tasks.change_usertaskstatus
+* user_tasks.delete_usertaskstatus
+* user_tasks.view_usertaskstatus
+* user_tasks.change_usertaskartifact
+* user_tasks.delete_usertaskartifact
+* user_tasks.view_usertaskartifact
+
+These permissions can be managed via Django's default database-backed authorization implementation, but using
+an alternative authorization backend can be easier to manage and support object-level permissions (for example, to
+determine if a user has permission to view and cancel a particular task but not others).  There is an
+`Authorization grid on Django Packages`_ listing several such backends; the test suite for ``django-user-tasks`` uses
+the `rules`_ package to define rule-based permissions which require no additional database setup.  These rules can
+be used in other applications via :py:func:`user_tasks.rules.add_rules`, if desired.
+
+.. _Authorization grid on Django Packages: https://djangopackages.org/grids/g/authorization/
+.. _rules: https://github.com/dfunckt/django-rules
+
+Restriction of status and artifact listings in the REST API to only those which the requesting user has permission
+to view can be done via the ``USER_TASKS_ARTIFACT_FILTERS`` and ``USER_TASKS_STATUS_FILTERS`` settings.  See the
+:doc:`settings documentation <settings>` for more information on how those work.
+
+Artifact URL Access
+-------------------
+
+Although the permission checks above cover most attempts to interact with a task, the artifacts associated with it
+may be located at URLs with less restricted access:
+
+* :py:attr:`UserTaskArtifact.text` is actually stored as part of the model instance, and hence is covered by
+  whichever authorization backend you've chosen to use.  But it's limited to text content only, and is not recommended
+  for large amounts of data.
+* :py:attr:`UserTaskArtifact.url` has no inherent security; the data at that URL is only as secure as the access
+  restrictions placed on it by the hosting system (which may or may not be the same service which is using
+  ``django-user-tasks``).  The URL may be hard to guess, but once discovered might be accessible to users other than
+  those with view permission for the artifact instance unless appropriate measures are taken.
+* :py:attr:`UserTaskArtifact.file` uses URLs generated by the Django file storage system in use.  The default
+  implementation imposes no particular access restrictions on the generated files, but alternatives with better
+  security are available.  For example, if you use the ``s3boto`` or ``s3boto3`` backend from `django-storages`_ with
+  a private Amazon S3 bucket, the artifact content URL will be presigned with query parameters which allow access to
+  the file for a limited time (determined by the ``AWS_QUERYSTRING_EXPIRE`` setting), so the content cannot be
+  accessed just by guessing the URL or gaining access to a previously served link with expired signature query
+  parameters.
+
+.. _django-storages: https://github.com/jschneier/django-storages

docs/conf.py

@@ -16,6 +16,8 @@
 from __future__ import absolute_import, unicode_literals
 
 import os
+import sys
+from subprocess import check_call
 
 # Configure Django for autodoc usage
 import django
@@ -459,6 +461,32 @@
 }
 
 
+def on_init(app):  # pylint: disable=unused-argument
+    """
+    Run sphinx-apidoc and swg2rst after Sphinx initialization.
+
+    Read the Docs won't run tox or custom shell commands, so we need this to
+    avoid checking in the generated reStructuredText files.
+    """
+    docs_path = os.path.abspath(os.path.dirname(__file__))
+    root_path = os.path.abspath(os.path.join(docs_path, '..'))
+    apidoc_path = 'sphinx-apidoc'
+    swg2rst_path = 'swg2rst'
+    if hasattr(sys, 'real_prefix'):  # Check to see if we are in a virtualenv
+        # If we are, assemble the path manually
+        bin_path = os.path.abspath(os.path.join(sys.prefix, 'bin'))
+        apidoc_path = os.path.join(bin_path, apidoc_path)
+        swg2rst_path = os.path.join(bin_path, swg2rst_path)
+    check_call([apidoc_path, '-o', docs_path, os.path.join(root_path, 'user_tasks'),
+                os.path.join(root_path, 'user_tasks/migrations')])
+    json_path = os.path.join(docs_path, 'swagger.json')
+    rst_path = os.path.join(docs_path, 'rest_api.rst')
+    check_call([swg2rst_path, json_path, '-f', 'rst', '-o', rst_path])
+
+
 def setup(app):
-    """Sphinx extension for applying some CSS overrides to the output theme."""
+    """
+    Sphinx extension: run sphinx-apidoc and apply some CSS overrides to the output theme.
+    """
+    app.connect('builder-inited', on_init)
     app.add_stylesheet('theme_overrides.css')

docs/data_cleanup.rst

@@ -0,0 +1,33 @@
+Data Cleanup
+============
+
+After running django-user-tasks for a while, you'll typically have an
+assortment of :py:class:`UserTaskStatus` and :py:class:`UserTaskArtifact`
+records sitting around in the database.  These can be either explicitly
+removed or automatically cleaned up.
+
+Explicit
+--------
+
+A :py:class:`UserTaskStatus` and any associated :py:class:`UserTaskArtifact`
+instances can be deleted via the REST API by a user with appropriate
+permissions.  It's usually appropriate to put a button for this in the UI
+wherever a status is shown as long as the task has reached a final state:
+``Canceled``, ``Failed``, or ``Succeeded``.  Note that if you delete a
+status record before the task has finished processing, it may be recreated
+upon the next state transition.
+
+.. _automatic-cleanup:
+
+Automatic
+---------
+
+:py:func:`user_tasks.tasks.purge_old_user_tasks` is a Celery task which
+deletes any :py:class:`UserTaskArtifact` records (and any associated
+artifacts) which were created more than a certain duration ago.  The
+task can be run explicitly, but you'd typically want it to run periodically
+by adding it to Celery's ``CELERYBEAT_SCHEDULE`` setting.
+
+The maximum age for status records defaults to 30 days, but can be
+customized by assigning a suitable ``timedelta`` to the
+``USER_TASKS_MAX_AGE`` setting.

docs/getting_started.rst

@@ -7,7 +7,7 @@ for a single type of task:
 1. Install ``django-user-tasks`` (typically via
    ``pip install django-user-tasks``, but use whatever mechanism you've chosen
    for dependency management).
-2. Add ``django-user-tasks`` to the ``INSTALLED_APPS`` Django setting.
+2. Add ``django-user-tasks`` and ``rest_framework`` to the ``INSTALLED_APPS`` Django setting.
 3. Run migrations to create the required database tables.
 4. Create a subclass of :py:class:`user_tasks.tasks.UserTask` and override one or
    two of its methods.
@@ -75,3 +75,29 @@ Now the name of the task includes the name of the course being exported,
 there's an indication of how far along execution of the task has progressed,
 and while in progress the status reflects the name of the section currently
 being exported.
+
+URL Configuration
+-----------------
+
+Out of the box, ``django-user-tasks`` provides a ``urls`` module containing a URLconf which places the REST API
+endpoints under ``tasks/`` and ``artifacts/``.  You can include ``user_tasks.urls.urlpatterns`` in your
+service's overall URL configuration, or create a custom configuration which uses paths of your choice.  For example:
+
+.. code-block:: python
+
+    from rest_framework.routers import SimpleRouter
+    from user_tasks.views import ArtifactViewSet, StatusViewSet
+
+    ROUTER = SimpleRouter()
+    ROUTER.register(r'user_task_artifacts', ArtifactViewSet, base_name='usertaskartifact')
+    ROUTER.register(r'user_tasks/', StatusViewSet, base_name='usertaskstatus')
+
+    urlpatterns = ROUTER.urls
+
+Task Status Signal
+------------------
+
+When a subclass of :py:class:`user_tasks.tasks.UserTaskMixin` reaches any end state (``Canceled``, ``Failed``, or
+``Succeeded``), a ``user_tasks.user_task_stopped`` signal is sent.  Listeners can use this signal to notify users of
+the status change, log relevant statistics, etc.  The signal's ``sender`` is the :py:class:`UserTaskStatus` class,
+and its ``status`` argument is the instance of that class for which the signal was sent.

docs/index.rst

@@ -14,7 +14,11 @@ Contents:
 
    readme
    getting_started
+   authorization
+   rest_api
+   data_cleanup
    linked_tasks
+   settings
    modules
    testing
    internationalization

docs/settings.rst

@@ -0,0 +1,6 @@
+Settings
+========
+
+.. autoclass:: user_tasks.conf.LazySettings
+   :members:
+   :noindex: