BibDocFile: Copyright and license for files

* Adds possibility to add copyright and license for each file (separate
  from copyright and license for whole record (closes inveniosoftware#1684)

* Modifies the filter_field_instances function in bibrecord:

  * Adds new mode to filter_mode that allows to filter all fields
    without specific subfield

* Adds functions (inside bibupload) to synchronize MARC for 542 and 540
  field with copyright and licenses from BibDocs, whenever bibupload
  with "FIX-MARC" field is run

* Adds subfield $8 to 8564 MARC field, that stores the id of a BibDoc

* FFT now modifies the copyright and license:

  * Added functions to bibupload, so when FFT is uploaded, it also
    allows to modify the copyright and license

* Fixed the regression tests (bibupload, websearch) after adding the
  subfield $8 to the 856 field

Signed-off-by: Sebastian Witowski <[email protected]>

config/invenio.conf

@@ -940,6 +940,8 @@ CFG_BIBDOCFILE_DOCUMENT_FILE_MANAGER_RESTRICTIONS = [
 CFG_BIBDOCFILE_DOCUMENT_FILE_MANAGER_MISC = {
     'can_revise_doctypes': ['*'],
     'can_comment_doctypes': ['*'],
+    'can_change_copyright_doctypes': ['*'],
+    'can_change_advanced_copyright_doctypes': ['*'],
     'can_describe_doctypes': ['*'],
     'can_delete_doctypes': ['*'],
     'can_keep_doctypes': ['*'],

modules/bibdocfile/lib/bibdocfile_config.py

@@ -22,6 +22,8 @@
 except ImportError:
     CFG_BIBDOCFILE_DOCUMENT_FILE_MANAGER_MISC = {
         'can_revise_doctypes': ['*'],
+        'can_change_copyright_doctypes': ['*'],
+        'can_change_advanced_copyright_doctypes': ['*'],
         'can_comment_doctypes': ['*'],
         'can_describe_doctypes': ['*'],
         'can_delete_doctypes': ['*'],

modules/bibrecord/lib/bibrecord.py

@@ -212,6 +212,8 @@ def filter_field_instances(field_instances, filter_subcode, filter_value, filter
         'e' - looking for exact match in subfield value
         's' - looking for substring in subfield value
         'r' - looking for regular expression in subfield value
+        'n' - looking for fields where subfield doesn't exist (this mode
+              ignores the filter_value)
 
         Example:
         record_filter_field(record_get_field_instances(rec, '999', '%', '%'), 'y', '2001')
@@ -222,7 +224,7 @@ def filter_field_instances(field_instances, filter_subcode, filter_value, filter
         @type filter_subcode: string
         @param filter_value: value of the subfield
         @type filter_value: string
-        @param filter_mode: 'e','s' or 'r'
+        @param filter_mode: 'e','s', 'r' or 'n'
     """
     matched = []
     if filter_mode == 'e':
@@ -245,6 +247,11 @@ def filter_field_instances(field_instances, filter_subcode, filter_value, filter
                    reg_exp.match(subfield[1]) is not None:
                     matched.append(instance)
                     break
+    elif filter_mode == 'n':
+        for instance in field_instances:
+            if filter_subcode not in [subfield[0] for subfield in instance[0]]:
+                matched.append(instance)
+
     return matched
 
 def record_drop_duplicate_fields(record):

modules/bibupload/lib/bibupload_revisionverifier_regression_tests.py

@@ -1114,7 +1114,7 @@ def test_corrected_record_affected_tags(self):
         """Checks if corrected record has affected fields in hstRECORD table"""
         query = "SELECT affected_fields from hstRECORD where id_bibrec=12 ORDER BY job_date DESC"
         res = run_sql(query)
-        self.assertEqual(res[0][0], "005__%,8564_%,909C0%,909C1%,909C5%,909CO%,909CS%")
+        self.assertEqual(res[0][0], "005__%,540__%,542__%,8564_%,909C0%,909C1%,909C5%,909CO%,909CS%")
 
 
     def test_append_to_record_affected_tags(self):

modules/websearch/lib/websearch_regression_tests.py

@@ -1297,8 +1297,8 @@ def test_search_engine_python_api_textmarc_full(self):
 000000085 695__ $$9LANL EDS$$aHigh Energy Physics - Theory
 000000085 700__ $$aPorrati, Massimo
 000000085 700__ $$aZaffaroni, A
-000000085 8564_ $$s112828$$u%(siteurl)s/record/85/files/0212181.ps.gz
-000000085 8564_ $$s151257$$u%(siteurl)s/record/85/files/0212181.pdf
+000000085 8564_ $$874$$s112828$$u%(siteurl)s/record/85/files/0212181.ps.gz
+000000085 8564_ $$874$$s151257$$u%(siteurl)s/record/85/files/0212181.pdf
 000000085 859__ [email protected]
 000000085 909C4 $$c289-293$$pPhys. Lett. B$$v561$$y2003
 000000085 916__ $$sn$$w200251
@@ -1345,8 +1345,8 @@ def test_search_engine_python_api_textmarc_full(self):
 000000001 65017 $$2SzGeCERN$$aExperiments and Tracks
 000000001 6531_ $$aLEP
 000000001 8560_ [email protected]
-000000001 8564_ $$s1585244$$u%(siteurl)s/record/1/files/0106015_01.jpg
-000000001 8564_ $$s20954$$u%(siteurl)s/record/1/files/0106015_01.gif?subformat=icon$$xicon
+000000001 8564_ $$81$$s1585244$$u%(siteurl)s/record/1/files/0106015_01.jpg
+000000001 8564_ $$81$$s20954$$u%(siteurl)s/record/1/files/0106015_01.gif?subformat=icon$$xicon
 000000001 909C0 $$o0003717PHOPHO
 000000001 909C0 $$y2000
 000000001 909C0 $$b81
@@ -1520,10 +1520,12 @@ def test_search_engine_python_api_xmlmarc_full(self):
     <subfield code="a">Zaffaroni, A</subfield>
   </datafield>
   <datafield tag="856" ind1="4" ind2=" ">
+    <subfield code="8">74</subfield>
     <subfield code="s">112828</subfield>
     <subfield code="u">%(siteurl)s/record/85/files/0212181.ps.gz</subfield>
   </datafield>
   <datafield tag="856" ind1="4" ind2=" ">
+    <subfield code="8">74</subfield>
     <subfield code="s">151257</subfield>
     <subfield code="u">%(siteurl)s/record/85/files/0212181.pdf</subfield>
   </datafield>
@@ -1739,10 +1741,12 @@ def test_search_engine_python_api_xmlmarc_full(self):
     <subfield code="f">[email protected]</subfield>
   </datafield>
   <datafield tag="856" ind1="4" ind2=" ">
+    <subfield code="8">1</subfield>
     <subfield code="s">1585244</subfield>
     <subfield code="u">%(siteurl)s/record/1/files/0106015_01.jpg</subfield>
   </datafield>
   <datafield tag="856" ind1="4" ind2=" ">
+    <subfield code="8">1</subfield>
     <subfield code="s">20954</subfield>
     <subfield code="u">%(siteurl)s/record/1/files/0106015_01.gif?subformat=icon</subfield>
     <subfield code="x">icon</subfield>
@@ -2108,8 +2112,8 @@ def test_search_engine_web_api_textmarc_full(self):
 000000085 695__ $$9LANL EDS$$aHigh Energy Physics - Theory
 000000085 700__ $$aPorrati, Massimo
 000000085 700__ $$aZaffaroni, A
-000000085 8564_ $$s112828$$u%(siteurl)s/record/85/files/0212181.ps.gz
-000000085 8564_ $$s151257$$u%(siteurl)s/record/85/files/0212181.pdf
+000000085 8564_ $$874$$s112828$$u%(siteurl)s/record/85/files/0212181.ps.gz
+000000085 8564_ $$874$$s151257$$u%(siteurl)s/record/85/files/0212181.pdf
 000000085 859__ [email protected]
 000000085 909C4 $$c289-293$$pPhys. Lett. B$$v561$$y2003
 000000085 916__ $$sn$$w200251
@@ -2156,8 +2160,8 @@ def test_search_engine_web_api_textmarc_full(self):
 000000001 65017 $$2SzGeCERN$$aExperiments and Tracks
 000000001 6531_ $$aLEP
 000000001 8560_ [email protected]
-000000001 8564_ $$s1585244$$u%(siteurl)s/record/1/files/0106015_01.jpg
-000000001 8564_ $$s20954$$u%(siteurl)s/record/1/files/0106015_01.gif?subformat=icon$$xicon
+000000001 8564_ $$81$$s1585244$$u%(siteurl)s/record/1/files/0106015_01.jpg
+000000001 8564_ $$81$$s20954$$u%(siteurl)s/record/1/files/0106015_01.gif?subformat=icon$$xicon
 000000001 909C0 $$o0003717PHOPHO
 000000001 909C0 $$y2000
 000000001 909C0 $$b81
@@ -2318,10 +2322,12 @@ def test_search_engine_web_api_xmlmarc_full(self):
     <subfield code="a">Zaffaroni, A</subfield>
   </datafield>
   <datafield tag="856" ind1="4" ind2=" ">
+    <subfield code="8">74</subfield>
     <subfield code="s">112828</subfield>
     <subfield code="u">%(siteurl)s/record/85/files/0212181.ps.gz</subfield>
   </datafield>
   <datafield tag="856" ind1="4" ind2=" ">
+    <subfield code="8">74</subfield>
     <subfield code="s">151257</subfield>
     <subfield code="u">%(siteurl)s/record/85/files/0212181.pdf</subfield>
   </datafield>
@@ -2537,10 +2543,12 @@ def test_search_engine_web_api_xmlmarc_full(self):
     <subfield code="f">[email protected]</subfield>
   </datafield>
   <datafield tag="856" ind1="4" ind2=" ">
+    <subfield code="8">1</subfield>
     <subfield code="s">1585244</subfield>
     <subfield code="u">%(siteurl)s/record/1/files/0106015_01.jpg</subfield>
   </datafield>
   <datafield tag="856" ind1="4" ind2=" ">
+    <subfield code="8">1</subfield>
     <subfield code="s">20954</subfield>
     <subfield code="u">%(siteurl)s/record/1/files/0106015_01.gif?subformat=icon</subfield>
     <subfield code="x">icon</subfield>
@@ -2861,8 +2869,8 @@ def test_record_web_api_textmarc_full(self):
 000000085 695__ $$9LANL EDS$$aHigh Energy Physics - Theory
 000000085 700__ $$aPorrati, Massimo
 000000085 700__ $$aZaffaroni, A
-000000085 8564_ $$s112828$$u%(siteurl)s/record/85/files/0212181.ps.gz
-000000085 8564_ $$s151257$$u%(siteurl)s/record/85/files/0212181.pdf
+000000085 8564_ $$874$$s112828$$u%(siteurl)s/record/85/files/0212181.ps.gz
+000000085 8564_ $$874$$s151257$$u%(siteurl)s/record/85/files/0212181.pdf
 000000085 859__ [email protected]
 000000085 909C4 $$c289-293$$pPhys. Lett. B$$v561$$y2003
 000000085 916__ $$sn$$w200251
@@ -2962,10 +2970,12 @@ def test_record_web_api_xmlmarc_full(self):
     <subfield code="a">Zaffaroni, A</subfield>
   </datafield>
   <datafield tag="856" ind1="4" ind2=" ">
+    <subfield code="8">74</subfield>
     <subfield code="s">112828</subfield>
     <subfield code="u">%(siteurl)s/record/85/files/0212181.ps.gz</subfield>
   </datafield>
   <datafield tag="856" ind1="4" ind2=" ">
+    <subfield code="8">74</subfield>
     <subfield code="s">151257</subfield>
     <subfield code="u">%(siteurl)s/record/85/files/0212181.pdf</subfield>
   </datafield>