Working GUI, improved error reporting and parallel validation

Author: OskarPersson

ESSArch_Core/WorkflowEngine/util.py

@@ -55,6 +55,7 @@ def _create_step(parent_step, flow, ip, responsible, context=None):
 
             child_s = ProcessStep.objects.create(
                 name=flow_entry['name'],
+                parallel=flow_entry['parallel'],
                 parent_step=parent_step,
                 parent_step_pos=e_idx,
                 eager=parent_step.eager,

ESSArch_Core/config/urls.py

@@ -39,7 +39,6 @@
     ValidationFilesViewSet,
     ValidationViewSet,
     ValidatorViewSet,
-    ValidatorWorkflowViewSet,
 )
 from ESSArch_Core.ip.views import (
     ConsignMethodViewSet,
@@ -323,7 +322,6 @@
 router.register(r'conversion-tools', ConversionToolViewSet)
 router.register(r'features', FeatureViewSet, basename='features')
 router.register(r'validators', ValidatorViewSet, basename='validators')
-router.register(r'validator-workflows', ValidatorWorkflowViewSet, basename='validator-workflows')
 router.register(r'validations', ValidationViewSet)
 router.register(r'events', EventIPViewSet)
 router.register(r'event-types', EventTypeViewSet)

ESSArch_Core/fixity/serializers.py

@@ -20,24 +20,32 @@ class ValidatorWorkflowSerializer(serializers.Serializer):
 
     def validate_validators(self, validators):
         new_data = []
-        sub_context = {'information_package': self.context['request'].data.get('information_package', None)}
+        ip = self.context['request'].data.get('information_package', None)
+        sub_context = {'information_package': ip}
+        sub_context.update(self.context)
 
         for validator in validators:
             name = validator.pop('name')
             klass = get_validator(name)
-            options_serializer = klass.get_options_serializer_class()(
-                data=validator.pop('options', {}),
-                context=sub_context,
-            )
+
             serializer = klass.get_serializer_class()(
                 data=validator, context=sub_context,
             )
-
             serializer.is_valid(True)
-            options_serializer.is_valid(True)
-
             data = serializer.validated_data
             data['name'] = name
+
+            options_data = validator.pop('options', {})
+            options_context = {
+                'information_package': ip,
+                'base_data': data,
+            }
+            options_serializer = klass.get_options_serializer_class()(
+                data=options_data,
+                context=options_context,
+            )
+
+            options_serializer.is_valid(True)
             data['options'] = options_serializer.validated_data
 
             new_data.append(data)

ESSArch_Core/fixity/validation/backends/checksum.py

@@ -1,5 +1,5 @@
 import logging
-import traceback
+import os
 
 from django.utils import timezone
 from rest_framework import serializers
@@ -28,14 +28,52 @@ class ChecksumValidator(BaseValidator):
 
     label = 'Checksum Validator'
 
+    @classmethod
+    def get_form(cls):
+        return [
+            {
+                'key': 'path',
+                'type': 'input',
+                'templateOptions': {
+                    'label': 'Path to validate',
+                    'required': True,
+                }
+            },
+            {
+                'key': 'options.algorithm',
+                'type': 'select',
+                'defaultValue': 'SHA-256',
+                'templateOptions': {
+                    'label': 'Checksum algorithm',
+                    'required': True,
+                    'labelProp': 'name',
+                    'valueProp': 'value',
+                    'options': [
+                        {'name': 'MD5', 'value': 'MD5'},
+                        {'name': 'SHA-1', 'value': 'SHA-1'},
+                        {'name': 'SHA-224', 'value': 'SHA-224'},
+                        {'name': 'SHA-256', 'value': 'SHA-256'},
+                        {'name': 'SHA-384', 'value': 'SHA-384'},
+                        {'name': 'SHA-512', 'value': 'SHA-512'},
+                    ]
+                }
+            },
+            {
+                'key': 'options.expected',
+                'type': 'input',
+                'templateOptions': {
+                    'label': 'Checksum',
+                    'required': True,
+                }
+            },
+        ]
+
     class Serializer(BaseValidator.Serializer):
-        context = serializers.ChoiceField(choices=['checksum_str', 'checksum_file', 'xml_file'])
+        context = serializers.CharField(default='checksum_str')
         block_size = serializers.IntegerField(default=65536)
 
     class OptionsSerializer(BaseValidator.OptionsSerializer):
         expected = serializers.CharField()
-        rootdir = serializers.CharField(default='', allow_blank=True)
-        recursive = serializers.BooleanField(default=True)
         algorithm = serializers.ChoiceField(
             choices=['MD5', 'SHA-1', 'SHA-224', 'SHA-256', 'SHA-384', 'SHA-512'],
             default='SHA-256',
@@ -52,8 +90,14 @@ def __init__(self, *args, **kwargs):
 
     def validate(self, filepath, expected=None):
         logger.debug('Validating checksum of %s' % filepath)
+
+        if self.ip is not None:
+            relpath = os.path.relpath(filepath, self.ip.object_path)
+        else:
+            relpath = filepath
+
         val_obj = Validation.objects.create(
-            filename=filepath,
+            filename=relpath,
             time_started=timezone.now(),
             validator=self.__class__.__name__,
             required=self.required,
@@ -82,14 +126,14 @@ def validate(self, filepath, expected=None):
             actual_checksum = calculate_checksum(filepath, algorithm=self.algorithm, block_size=self.block_size)
             if actual_checksum != checksum:
                 raise ValidationError("checksum for %s is not valid (%s != %s)" % (
-                    filepath, checksum, actual_checksum
+                    relpath, checksum, actual_checksum
                 ))
             passed = True
-        except Exception:
-            val_obj.message = traceback.format_exc()
+        except Exception as e:
+            val_obj.message = str(e)
             raise
         else:
-            message = 'Successfully validated checksum of %s' % filepath
+            message = 'Successfully validated checksum of %s' % relpath
             val_obj.message = message
             logger.info(message)
         finally:

ESSArch_Core/fixity/validation/backends/xml.py

@@ -94,13 +94,19 @@ def __init__(self, *args, **kwargs):
             self.fields['context'] = FilePathField(ip.object_path, allow_blank=True, default='')
 
     class OptionsSerializer(BaseValidator.OptionsSerializer):
-        rootdir = serializers.CharField(default='', allow_blank=True)
+        rootdir = serializers.CharField(required=False)
         recursive = serializers.BooleanField(default=True)
         default_algorithm = serializers.ChoiceField(
             choices=['MD5', 'SHA-1', 'SHA-224', 'SHA-256', 'SHA-384', 'SHA-512'],
             default='SHA-256',
         )
 
+        def validate(self2, data):
+            if 'rootdir' not in data:
+                data['rootdir'] = self2.context['base_data']['path']
+
+            return data
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 

ESSArch_Core/fixity/views.py

@@ -1,9 +1,6 @@
-import os
-
-from django.db import transaction
 from django.db.models import Exists, Max, Min, OuterRef
 from django_filters.rest_framework import DjangoFilterBackend
-from rest_framework import filters, mixins, status, viewsets
+from rest_framework import filters, viewsets
 from rest_framework.permissions import IsAuthenticated
 from rest_framework.response import Response
 from rest_framework_extensions.mixins import NestedViewSetMixin
@@ -15,14 +12,11 @@
     ConversionToolSerializer,
     ValidationFilesSerializer,
     ValidationSerializer,
-    ValidatorWorkflowSerializer,
 )
 from ESSArch_Core.fixity.validation import (
     AVAILABLE_VALIDATORS,
     get_backend as get_validator,
 )
-from ESSArch_Core.WorkflowEngine.models import ProcessStep
-from ESSArch_Core.WorkflowEngine.util import create_workflow
 
 
 class ConversionToolViewSet(viewsets.ReadOnlyModelViewSet):
@@ -57,49 +51,6 @@ def list(self, request, format=None):
         return Response(validators)
 
 
-class ValidatorWorkflowViewSet(mixins.CreateModelMixin, viewsets.GenericViewSet):
-    queryset = ProcessStep.objects.all()
-    serializer_class = ValidatorWorkflowSerializer
-
-    def create(self, request, *args, **kwargs):
-        serializer = self.get_serializer(data=request.data, context={'request': self.request})
-        serializer.is_valid(raise_exception=True)
-        workflow_spec = []
-        ip = serializer.validated_data['information_package']
-
-        for validator in serializer.validated_data['validators']:
-            name = validator['name']
-            klass = get_validator(name)
-            options_serializer = klass.get_options_serializer_class()(data=validator.get('options', {}))
-            options_serializer.is_valid(raise_exception=True)
-            options = options_serializer.validated_data
-
-            path = os.path.join(ip.object_path, validator['path'])
-            options['rootdir'] = ip.object_path
-
-            task_spec = {
-                'name': 'ESSArch_Core.fixity.validation.tasks.Validate',
-                'label': 'Validate using {}'.format(klass.label),
-                'args': [name, path],
-                'params': {'context': validator['context'], 'options': options},
-            }
-
-            workflow_spec.append(task_spec)
-
-        with transaction.atomic():
-            step = {
-                'step': True,
-                'name': serializer.validated_data['purpose'],
-                'children': workflow_spec
-            }
-            workflow = create_workflow([step], ip=ip, name='Validation')
-
-        workflow.run()
-
-        headers = self.get_success_headers(serializer.data)
-        return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)
-
-
 class ValidationViewSet(NestedViewSetMixin, viewsets.ReadOnlyModelViewSet):
     queryset = Validation.objects.all().order_by('filename', 'validator')
     serializer_class = ValidationSerializer

ESSArch_Core/ip/views.py

@@ -64,8 +64,12 @@
 from ESSArch_Core.essxml.util import get_objectpath, parse_submit_description
 from ESSArch_Core.exceptions import Conflict, NoFileChunksFound
 from ESSArch_Core.fixity.format import FormatIdentifier
+from ESSArch_Core.fixity.serializers import ValidatorWorkflowSerializer
 from ESSArch_Core.fixity.transformation import AVAILABLE_TRANSFORMERS
-from ESSArch_Core.fixity.validation import AVAILABLE_VALIDATORS
+from ESSArch_Core.fixity.validation import (
+    AVAILABLE_VALIDATORS,
+    get_backend as get_validator,
+)
 from ESSArch_Core.fixity.validation.backends.checksum import ChecksumValidator
 from ESSArch_Core.ip.filters import (
     AgentFilter,
@@ -1923,46 +1927,45 @@ def unlock_profile(self, request, pk=None):
             )
         })
 
-    @transaction.atomic
     @action(detail=True, methods=['post'], url_path='validate')
     def validate(self, request, pk=None):
         ip = self.get_object()
 
-        prepare = Path.objects.get(entity="ingest_workarea").value
-        xmlfile = os.path.join(prepare, "%s.xml" % pk)
+        request.data['information_package'] = str(ip.pk)
+        serializer = ValidatorWorkflowSerializer(data=request.data, context={'request': request})
+        serializer.is_valid(raise_exception=True)
+        workflow_spec = []
+        ip = serializer.validated_data['information_package']
 
-        step = ProcessStep.objects.create(
-            name="Validation",
-            information_package=ip
-        )
+        for validator in serializer.validated_data['validators']:
+            name = validator['name']
+            klass = get_validator(name)
 
-        step.add_tasks(
-            ProcessTask.objects.create(
-                name="ESSArch_Core.tasks.ValidateXMLFile",
-                params={
-                    "xml_filename": xmlfile
-                },
-                log=EventIP,
-                information_package=ip,
-                responsible=self.request.user,
-            ),
-            ProcessTask.objects.create(
-                name="ESSArch_Core.tasks.ValidateFiles",
-                params={
-                    "mets_path": xmlfile,
-                    "validate_fileformat": True,
-                    "validate_integrity": True,
-                },
-                log=EventIP,
-                processstep_pos=0,
-                information_package=ip,
-                responsible=self.request.user,
-            )
-        )
+            options = validator['options']
+            path = os.path.join(ip.object_path, validator['path'])
 
-        step.run()
+            task_spec = {
+                'name': 'ESSArch_Core.fixity.validation.tasks.Validate',
+                'label': 'Validate using {}'.format(klass.label),
+                'args': [name, path],
+                'params': {'context': validator['context'], 'options': options},
+            }
 
-        return Response("Validating IP")
+            workflow_spec.append(task_spec)
+
+        with transaction.atomic():
+            step = {
+                'step': True,
+                'name': serializer.validated_data['purpose'],
+                'parallel': True,
+                'children': workflow_spec
+            }
+            workflow = create_workflow([step], ip=ip, name='Validation')
+
+        workflow.run()
+
+        headers = self.get_success_headers(serializer.data)
+        return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)
 
     def update(self, request, *args, **kwargs):
         ip = self.get_object()