[script][m]: - script to modify invalid datapackage.json

Author: Mikanebu

examples/modify.py

@@ -0,0 +1,86 @@
+import json
+
+# This script modifies date type format to "any" in fields list. 
+# Also, changes unsupported number types to "number"
+# Plus, modifies ojbects to list of objects 
+
+class Modify(object):
+    def __init__(self, path='datapackage.json'):
+        self.path = path
+
+    def transform_package(self, data):
+        '''Transform data
+        '''
+        unsupported_number_types = [
+            'decimal', 'double', 'float', 'binary'
+        ]
+        for item in data:
+            if type(data[item]) is dict:
+                data[item] = [data[item]]
+        for res_idx in range (len(data['resources'])):
+            fields = data['resources'][res_idx]['schema']['fields']
+            for field_idx in range(len(fields)):
+                field = data['resources'][res_idx]['schema']['fields'][field_idx]
+                if field['type'] == 'date':
+                    data['resources'][res_idx]['schema']['fields'][field_idx]['format'] = "any"
+                if field['type'] in unsupported_number_types:
+                    data['resources'][res_idx]['schema']['fields'][field_idx]['type'] = "number"
+        return data
+
+    def modify(self):
+        '''Rewrites datapackage.json
+        '''
+        transformed_data = self.transform_package(json.load(open(self.path)))
+        jsonfile = open(self.path , 'w')
+        json.dump(transformed_data, jsonfile,  indent=2)
+
+    def show(self):
+        '''See modified datapackage.json as a json string
+        '''
+        transformed_data = self.transform_package(json.load(open(self.path)))
+        print (json.dumps(transformed_data, indent=2))
+
+## ==============================================
+## CLI
+
+import sys
+import optparse
+import inspect
+
+def _object_methods(obj):
+    methods = inspect.getmembers(obj, inspect.ismethod)
+    methods = filter(lambda (name, y): not name.startswith('_'), methods)
+    methods = dict(methods)
+    return methods
+
+def _main(functions_or_object):
+    isobject = inspect.isclass(functions_or_object)
+    if isobject:
+        _methods = _object_methods(functions_or_object)
+    else:
+        _methods = _module_functions(functions_or_object)
+
+    usage = '''%prog {action}
+Actions:
+    '''
+    usage += '\n    '.join(
+        [ '%s: %s' % (name, m.__doc__.split('\n')[0] if m.__doc__ else '') for (name,m)
+        in sorted(_methods.items()) ])
+    parser = optparse.OptionParser(usage)
+    # Optional: for a config file
+    # parser.add_option('-c', '--config', dest='config',
+    #         help='Config file to use.')
+    options, args = parser.parse_args()
+
+    if not args or not args[0] in _methods:
+        parser.print_help()
+        sys.exit(1)
+
+    method = args[0]
+    if isobject:
+        getattr(functions_or_object(), method)(*args[1:])
+    else:
+        _methods[method](*args[1:])
+
+if __name__ == '__main__':
+    _main(Modify)

examples/modify_test.py

@@ -0,0 +1,62 @@
+import unittest
+from modify import Modify
+
+class HelpersTestCase(unittest.TestCase):
+    def setUp(self):
+        self.to_modify = Modify()
+
+    def test_modify_date_format(self):
+        expected = "any"
+        dp = {'resources': [{"schema": {"fields": [{"name": "test","type": "date"}]}}]}
+        datapackage = self.to_modify.transform_package(dp)
+        result = datapackage['resources'][0]['schema']['fields'][0]['format']
+        self.assertEqual(result, expected)
+
+    def test_modify_date_format_works_with_multiple_date_fields(self):
+        expected = "any"
+        dp = {
+            'resources': [{
+            "schema": {"fields": [
+                {"name": "test","type": "date"},
+                {"name": "another-test","type": "date", "format": 'YYYY'}
+                ]}
+            }]
+        }
+        datapackage = self.to_modify.transform_package(dp)
+        result_1 = datapackage['resources'][0]['schema']['fields'][0]['format']
+        result_2 = datapackage['resources'][0]['schema']['fields'][1]['format']
+        self.assertEqual(result_1, expected)
+        self.assertEqual(result_2, expected)
+
+    def test_modify_date_format_works_if_format_is_wrong(self):
+        expected = "any"
+        dp = {
+            'resources': [{
+            "schema": {"fields": [{"name": "test","type": "date", "format": 'YYYY'}]}
+            }]
+        }
+        datapackage = self.to_modify.transform_package(dp)
+        result = datapackage['resources'][0]['schema']['fields'][0]['format']
+        self.assertEqual(result, expected)
+
+    def test_transform_works_if_number_type_is_wrong(self):
+        expected = 'number'
+        dp = {'resources': [{"schema": {"fields": [{"name": "test","type": "decimal"}]}}]}
+        datapackage = self.to_modify.transform_package(dp)
+        result = datapackage['resources'][0]['schema']['fields'][0]['type']
+        self.assertEqual(result, expected)
+
+        dp = {'resources': [{"schema": {"fields": [{"name": "test","type": "float"}]}}]}
+        datapackage = self.to_modify.transform_package(dp)
+        result = datapackage['resources'][0]['schema']['fields'][0]['type']
+        self.assertEqual(result, expected)
+
+    def test_turns_into_list_if_datapackage_keys_are_objects(self):
+        dp = {
+            'resources': {"schema": {"fields": [{"name": "test","type": "decimal"}]}},
+            'licenses': { "id": "test", "title": "test-license", "url": "test.com"},
+            'sources': {'name': 'test', 'web': 'test.com'}
+        }
+        datapackage = self.to_modify.transform_package(dp)
+        self.assertEqual(type(datapackage['resources']), list)
+        self.assertEqual(type(datapackage['licenses']), list)