Remove the completed field from upload and refactor serializer

[noissue]

CHANGES/4982.feature

@@ -0,0 +1 @@
+Allow users to pass sha256 with each chunk to have Pulp verify the chunk.

CHANGES/5087.bugfix

@@ -0,0 +1 @@
+Have the commit endpoint dispatch a task to create artifacts from chunked uploads

CHANGES/5087.removal

@@ -0,0 +1 @@
+Removed upload parameter from artifact create endpoint and converted upload commit to return 202.

CHANGES/5150.feature

@@ -0,0 +1 @@
+Users can view chunks info for chunked uploads in the API

docs/workflows/upload-publish.rst

@@ -14,18 +14,18 @@ This returns an upload href (e.g. ``/pulp/api/v3/uploads/a8b5a7f7-2f22-460d-ab20
 be used for chunks. Chunks can be uploaded in any order or in parallel::
 
     http --form PUT :24817/pulp/api/v3/uploads/a8b5a7f7-2f22-460d-ab20-d5616cb71cdd/ file@./chunk2 'Content-Range:bytes 6291456-10485759/*'
-    http --form PUT :24817/pulp/api/v3/uploads/a8b5a7f7-2f22-460d-ab20-d5616cb71cdd/ file@./chunk1 'Content-Range:bytes 0-6291455'
+    http --form PUT :24817/pulp/api/v3/uploads/a8b5a7f7-2f22-460d-ab20-d5616cb71cdd/ file@./chunk1 'Content-Range:bytes 0-6291455/*'
 
 Note: You can send an optional sha256 argument::
 
-    http --form PUT :24817/pulp/api/v3/uploads/a8b5a7f7-2f22-460d-ab20-d5616cb71cdd/ file@./chunk1 'Content-Range:bytes 0-6291455' sha256=7ffc86295de63e96006ce5ab379050628aa5d51f816267946c71906594e13870
+    http --form PUT :24817/pulp/api/v3/uploads/a8b5a7f7-2f22-460d-ab20-d5616cb71cdd/ file@./chunk1 'Content-Range:bytes 0-6291455/*' sha256=7ffc86295de63e96006ce5ab379050628aa5d51f816267946c71906594e13870
 
-Once all chunks have been uploaded, a final POST request with the file md5 can be sent to complete the
-upload::
+Once all chunks have been uploaded, a final POST request with the file sha256 can be sent to
+complete the upload::
 
     http POST :24817/pulp/api/v3/uploads/a8b5a7f7-2f22-460d-ab20-d5616cb71cdd/commit sha256=abc123...
 
-Note that after creating an artifact from an upload, the upload gets deleted and cannot be re-used.
+This queues a task that creates an artifact, and the upload gets deleted and cannot be re-used.
 
 Putting this altogether, here is an example that uploads a 1.iso file in two chunks::
 

pulpcore/app/migrations/0003_remove_upload_completed.py

@@ -0,0 +1,17 @@
+# Generated by Django 2.2.3 on 2019-07-24 17:28
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0002_increase_artifact_size_field'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='upload',
+            name='completed',
+        ),
+    ]

pulpcore/app/models/upload.py

@@ -16,12 +16,10 @@ class Upload(Model):
 
         file (models.FileField): The stored file.
         size (models.BigIntegerField): The size of the file in bytes.
-        completed (models.DateTimeField): Time when the upload is committed
     """
 
     file = models.FileField(null=False, max_length=255)
     size = models.BigIntegerField()
-    completed = models.DateTimeField(null=True)
 
     def append(self, chunk, offset, sha256=None):
         """

pulpcore/app/serializers/__init__.py

@@ -15,7 +15,6 @@
 )
 from .content import (  # noqa
     ArtifactSerializer,
-    ArtifactUploadSerializer,
     ContentChecksumSerializer,
     MultipleArtifactContentSerializer,
     NoArtifactContentSerializer,

pulpcore/app/serializers/content.py

@@ -5,7 +5,7 @@
 from rest_framework import serializers
 from rest_framework.validators import UniqueValidator
 
-from pulpcore.app import files, models
+from pulpcore.app import models
 from pulpcore.app.serializers import base, fields
 
 UNIQUE_ALGORITHMS = ['sha256', 'sha384', 'sha512']
@@ -201,15 +201,6 @@ def validate(self, data):
         """
         super().validate(data)
 
-        if ('file' not in data and 'upload' not in data) or \
-                ('file' in data and 'upload' in data):
-            raise serializers.ValidationError(_("Either 'file' or 'upload' parameter must be "
-                                                "supplied but not both."))
-
-        if 'upload' in data:
-            self.upload = data.pop('upload')
-            data['file'] = files.PulpTemporaryUploadedFile.from_file(self.upload.file.file)
-
         if 'size' in data:
             if data['file'].size != int(data['size']):
                 raise serializers.ValidationError(_("The size did not match actual size of file."))
@@ -234,35 +225,7 @@ def validate(self, data):
                     validator(digest)
         return data
 
-    def create(self, validated_data):
-        """
-        Create the artifact and delete its associated upload (if there is one)
-
-        Args:
-            validated_data (dict): Data to save to the database
-        """
-        artifact = super().create(validated_data)
-        if hasattr(self, 'upload'):
-            # creating an artifact will move the upload file so we need to delete the db record
-            self.upload.delete()
-        return artifact
-
     class Meta:
         model = models.Artifact
         fields = base.ModelSerializer.Meta.fields + ('file', 'size', 'md5', 'sha1', 'sha224',
                                                      'sha256', 'sha384', 'sha512')
-
-
-class ArtifactUploadSerializer(ArtifactSerializer):
-    upload = serializers.HyperlinkedRelatedField(
-        help_text=_("An href for an Upload."),
-        view_name="uploads-detail",
-        write_only=True,
-        required=False,
-        queryset=models.Upload.objects.exclude(completed__isnull=True)
-    )
-
-    class Meta:
-        model = models.Artifact
-        fields = base.ModelSerializer.Meta.fields + ('file', 'size', 'md5', 'sha1', 'sha224',
-                                                     'sha256', 'sha384', 'sha512', 'upload')

pulpcore/app/tasks/upload.py

@@ -1,46 +1,35 @@
-from datetime import datetime
 from gettext import gettext as _
 from logging import getLogger
 
-from rest_framework import serializers
-
-from pulpcore.app import models
+from pulpcore.app import files, models
 from pulpcore.app.models import CreatedResource
-from pulpcore.app.serializers import ArtifactUploadSerializer
-from pulpcore.tasking.util import get_url
+from pulpcore.app.serializers import ArtifactSerializer
 
 log = getLogger(__name__)
 
 
 def commit(upload_id, sha256):
     """
-    Commit the upload and mark it as completed.
-
-    Commit a :class:`~pulpcore.app.models.Upload`
+    Commit the upload and turn it into an artifact.
 
     Args:
         upload_id (int): The upload primary key
         sha256 (str): The checksum for the uploaded file
-
     """
     try:
         upload = models.Upload.objects.get(pk=upload_id)
     except models.Upload.DoesNotExist:
         log.info(_('The upload was not found. Nothing to do.'))
         return
 
-    log.info(_('Commiting the upload %(i)d and mark it as completed'), {'i': upload_id})
-
-    if not sha256:
-        raise serializers.ValidationError(_("Checksum not supplied."))
-
-    upload.completed = datetime.now()
-    upload.save()
-
-    data = {'upload': get_url(upload), 'sha256': sha256}
-    serializer = ArtifactUploadSerializer(data=data)
+    file = files.PulpTemporaryUploadedFile.from_file(upload.file.file)
+    data = {'file': file, 'sha256': sha256}
+    serializer = ArtifactSerializer(data=data)
     serializer.is_valid(raise_exception=True)
     artifact = serializer.save()
 
     resource = CreatedResource(content_object=artifact)
     resource.save()
+
+    # delete the upload since it can't be reused to create another artifact
+    upload.delete()

pulpcore/app/viewsets/upload.py

@@ -67,9 +67,6 @@ def update(self, request, pk=None):
         """
         upload = self.get_object()
 
-        if upload.completed is not None:
-            raise serializers.ValidationError(_("Cannot upload chunk for a completed upload."))
-
         try:
             chunk = request.data['file']
         except KeyError:
@@ -100,7 +97,7 @@ def update(self, request, pk=None):
     @detail_route(methods=('post',))
     def commit(self, request, pk):
         """
-        Generates a Task to commit the upload and mark it as completed.
+        Generates a Task to commit the upload and create an artifact
         """
         try:
             sha256 = request.data['sha256']

pulpcore/tests/functional/api/test_upload.py

@@ -122,7 +122,7 @@ def test_upload_response(self):
             UPLOAD_PATH, {'size': self.size_file}
         )
 
-        expected_keys = ['_href', '_created', 'size', 'completed']
+        expected_keys = ['_href', '_created', 'size']
 
         self.assertEqual([*upload_request], expected_keys, upload_request)