-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathplugin.py
1801 lines (1509 loc) · 71.3 KB
/
plugin.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/python
# -*- coding: utf-8 -*-
import ast
import configparser
import csv
import datetime
import json
import logging
import os
import sys
import urllib
import xml.etree.ElementTree as ET
import idutils
import numpy as np
import pandas as pd
import requests
from dicttoxml import dicttoxml
import api.utils as ut
from api.evaluator import ConfigTerms, EvaluatorBase, MetadataValuesBase
from api.vocabulary import Vocabulary
logging.basicConfig(
stream=sys.stdout, level=logging.DEBUG, format="'%(name)s:%(lineno)s' | %(message)s"
)
logger = logging.getLogger("api.plugin.evaluation_steps")
logger_api = logging.getLogger("api.plugin")
class MetadataValues(MetadataValuesBase):
@classmethod
def _get_identifiers_metadata(cls, element_values):
"""Get the list of identifiers for the metadata.
* Format EPOS DEV API:
"id": "77c89ce5-cbaa-4ea8-bcae-fdb1da932f6e"
"""
return [element_values]
@classmethod
def _get_identifiers_data(cls, element_values):
"""Get the list of identifiers for the data.
* Format EPOS DEV API:
"identifiers": [{
"type": "DOI",
"value": "https://doi.org/10.13127/tsunami/neamthm18"
}]
"""
return [value_data["value"] for value_data in element_values]
@classmethod
def _get_formats(cls, element_values):
"""Return the list of formats defined through <availableFormats> metadata
attribute.
* Format EPOS PROD & DEV API:
"availableFormats": [{
"format": "SHAPE-ZIP",
"href": "https://www.ics-c.epos-eu.org/api/v1/execute/b8b5f0c3-a71c-448e-88ac-3a3c5d97b08f?format=SHAPE-ZIP",
"label": "SHAPE-ZIP",
"originalFormat": "SHAPE-ZIP",
"type": "ORIGINAL"
}]
"""
return list(
filter(
None, [value_data.get("format", "") for value_data in element_values]
)
)
@classmethod
def _get_temporal_coverage(cls, element_values):
"""Return a list of tuples with temporal coverages for start and end date.
* Format EPOS PROD & DEV API:
"temporalCoverage": [{
"startDate": "2018-01-31T00:00:00Z"
}]
"""
return [
{
"start_date": value_data.get("startDate", ""),
"end_date": value_data.get("endDate", ""),
}
for value_data in element_values
]
@classmethod
def _get_person(cls, element_values):
"""Return a list with person-related info.
* Format EPOS DEV API:
"contactPoints": [{
"id": "8069667d-7676-4c02-b98e-b1e044ab4cd7",
"metaid": "2870c8e4-c616-4eaf-b84d-502f6a3ee2fb",
"uid": "http://orcid.org/0000-0003-4551-3339/Contact"
}]
"""
person_data = [
value_data["person"].get("uid", "")
for value_data in element_values
if "person" in list(value_data)
]
if not person_data:
logger_api.debug(
"Could not fetch person contacts through 'person:uid' property. Trying to parse directly with 'uid' property"
)
person_data = [
ut.get_orcid_str(value_data.get("uid", ""))
for value_data in element_values
]
return person_data
def _validate_format(self, formats, vocabularies, plugin_obj):
from fair import app_dirname
formats_data = {}
for vocabulary_id, vocabulary_url in vocabularies.items():
# Store successfully validated licenses, grouped by CV
formats_data[vocabulary_id] = {"valid": [], "non_valid": []}
# imtypes (IANA Media Types)
if vocabulary_id in ["imtypes"]:
logger_api.debug(
"Validating formats according to IANA Media Types vocabulary: %s"
% formats
)
iana_formats = plugin_obj.vocabulary.get_iana_media_types()
# Compare with given input formats
for _format in formats:
if _format.lower() in iana_formats:
logger.debug(
"Format complies with IANA Internet Media Types vocabulary: %s"
% _format
)
formats_data[vocabulary_id]["valid"].append(_format)
else:
logger.warning(
"Format does not comply with IANA Internet Media Types vocabulary: %s"
% _format
)
formats_data[vocabulary_id]["non_valid"].append(_format)
return formats_data
def _get_license(self, element_values):
"""Return a list of licenses.
* Format EPOS PROD & DEV API:
"license": "https://spdx.org/licenses/CC-BY-4.0.html"
"""
if isinstance(element_values, str):
logger.debug(
"Provided licenses as a string for metadata element <license>: %s"
% element_values
)
return [element_values]
elif isinstance(element_values, list):
logger.debug(
"Provided licenses as a list for metadata element <license>: %s"
% element_values
)
return element_values
def _validate_license(self, licenses, vocabularies, machine_readable=False):
license_data = {}
for vocabulary_id, vocabulary_url in vocabularies.items():
# Store successfully validated licenses, grouped by CV
license_data[vocabulary_id] = {"valid": [], "non_valid": []}
# SPDX
if vocabulary_id in ["spdx"]:
logger_api.debug(
"Validating licenses according to SPDX vocabulary: %s" % licenses
)
for _license in licenses:
if ut.is_spdx_license(_license, machine_readable=machine_readable):
logger.debug(
"License successfully validated according to SPDX vocabulary: %s"
% _license
)
license_data[vocabulary_id]["valid"].append(_license)
else:
logger.warning(
"Could not find any license match in SPDX vocabulary for '%s'"
% _license
)
license_data[vocabulary_id]["non_valid"].append(_license)
else:
logger.warning(
"Validation of vocabulary '%s' not yet implemented" % vocabulary_id
)
return license_data
class Plugin(EvaluatorBase):
"""A class used to define FAIR indicators tests, tailored to the DT-GEO prototype
metadata catalog (EPOS ICS-C based)."""
def __init__(self, *args, **kwargs):
super().__init__(
*args,
**kwargs,
)
logger.debug("Using FAIR-EVA's plugin: %s" % self.name)
# Metadata gathering
metadata_sample = self.get_metadata()
self.metadata = pd.DataFrame(
metadata_sample,
columns=["metadata_schema", "element", "text_value", "qualifier"],
)
logger.debug(
"Obtained metadata from repository: %s" % (self.metadata.to_json())
)
# Protocol for (meta)data accessing
if len(self.metadata) > 0:
self.access_protocols = ["http"]
# Config attributes
self.terms_map = ast.literal_eval(self.config[self.name]["terms_map"])
self.metadata_access_manual = ast.literal_eval(
self.config[self.name]["metadata_access_manual"]
)
self.data_access_manual = ast.literal_eval(
self.config[self.name]["data_access_manual"]
)
self.terms_access_protocols = ast.literal_eval(
self.config[self.name]["terms_access_protocols"]
)
self.dict_vocabularies = ast.literal_eval(
self.config[self.name]["dict_vocabularies"]
)
self.metadata_standard = ast.literal_eval(
self.config[self.name]["metadata_standard"]
)
self.metadata_authentication = ast.literal_eval(
self.config[self.name]["metadata_authentication"]
)
self.metadata_persistence = ast.literal_eval(
self.config[self.name]["metadata_persistence"]
)
# Metadata gathering
metadata_sample = self.get_metadata()
self.metadata = pd.DataFrame(
metadata_sample,
columns=["metadata_schema", "element", "text_value", "qualifier"],
)
if len(self.metadata) > 0:
logger.debug("Obtained metadata from repository: %s" % self.api_endpoint)
logger_api.debug(self.metadata)
else:
raise Exception(
"Could not get metadata information from repository: %s"
% self.api_endpoint
)
logger.debug("METADATA: %s" % (self.metadata))
@property
def metadata_utils(self):
return MetadataValues()
@staticmethod
def get_ids(api_endpoint, pattern_to_query=""):
url = api_endpoint + "/resources/search?facets=false&q=" + pattern_to_query
response_payload = ut.make_http_request(url=url)
results = response_payload.get("results", [])
return [
result["id"] for result in results["distributions"] if "id" in result.keys()
]
def get_metadata(self):
metadata_sample = []
eml_schema = "epos"
final_url = (
self.api_endpoint + "/resources/details/" + self.item_id + "?extended=true"
)
error_in_metadata = False
headers = {
"accept": "application/json",
}
response = requests.get(
final_url,
headers=headers,
)
if not response.ok:
msg = (
"Error while connecting to metadata repository: %s (status code: %s)"
% (response.url, response.status_code)
)
error_in_metadata = True
# headers
self.metadata_endpoint_headers = response.headers
logger.debug(
"Storing headers from metadata repository: %s"
% self.metadata_endpoint_headers
)
dicion = response.json()
if not dicion:
msg = (
"Error: empty metadata received from metadata repository: %s"
% final_url
)
error_in_metadata = True
if error_in_metadata:
logger.error(msg)
raise Exception(msg)
for key in dicion.keys():
if str(type(dicion[key])) == "<class 'dict'>":
q = dicion[key]
for key2 in q.keys():
metadata_sample.append([eml_schema, key2, q[key2], key])
if key == "relatedDataProducts":
q = dicion[key][0]
for key2 in q.keys():
if str(type(q[key2])) == "<class 'dict'>":
w = q[key2]
for key3 in w.keys():
metadata_sample.append([eml_schema, key3, w[key3], key2])
elif (
str(type(q[key2])) == "<class 'list'>"
and len(q[key2]) == 0
and str(type(q[key2][0])) == "<class 'dict'>"
):
w = q[key2][0]
for key3 in w.keys():
metadata_sample.append([eml_schema, key3, w[key3], key2])
else:
metadata_sample.append([eml_schema, key2, q[key2], key])
"""Elif str(type(dicion[key])) == "<class 'list'>" and:
q = dicion[key]
if str(type(q[0])) == "<class 'dict'>":
if len(q) ==1:
q=q[0]
for key2 in q.keys():
metadata_sample.append([eml_schema, key2, q[key2], key])
else:
for elem in q:
metadata_sample.append([eml_schema, key, elem, None])
"""
else:
metadata_sample.append([eml_schema, key, dicion[key], None])
return metadata_sample
@ConfigTerms(term_id="identifier_term")
def rda_f1_01m(self, **kwargs):
"""Indicator RDA-F1-01M: Metadata is identified by a persistent identifier.
This indicator is linked to the following principle: F1 (meta)data are assigned a globally
unique and eternally persistent identifier. More information about that principle can be found
here.
This indicator evaluates whether or not the metadata is identified by a persistent identifier.
A persistent identifier ensures that the metadata will remain findable over time, and reduces
the risk of broken links.
Parameters
----------
identifier_term : dict
A dictionary with metadata information about the identifier/s used for the metadata (see ConfigTerms class for further details)
Returns
-------
points
- 0/100 if no persistent identifier is used for the metadata
- 100/100 if a persistent identifier is used for the metadata
msg
Message with the results or recommendations to improve this indicator
"""
id_list = kwargs["Metadata Identifier"]
points, msg_list = self.eval_persistency(id_list, data_or_metadata="metadata")
logger.debug(msg_list)
if points == 0:
if self.metadata_persistence:
if self.check_link(self.metadata_persistence):
points = 100
msg = "Identifier found and persistence policy given "
return (points, {"message": msg, "points": points})
return (points, msg_list)
@ConfigTerms(term_id="identifier_term_data")
def rda_f1_01d(self, **kwargs):
"""Indicator RDA-F1-01D: Data is identified by a persistent identifier.
This indicator is linked to the following principle: F1 (meta)data are assigned a globally
unique and eternally persistent identifier. More information about that principle can be found
here.
This indicator evaluates whether or not the data is identified by a persistent identifier.
A persistent identifier ensures that the data will remain findable over time and reduces the
risk of broken links.
Parameters
----------
identifier_term_data : dict
A dictionary with metadata information about the identifier/s used for the data (see ConfigTerms class for further details)
Returns
-------
points
Returns a value (out of 100) that reflects the amount of data identifiers that are persistent.
msg
Message with the results or recommendations to improve this indicator
"""
id_list = kwargs["Data Identifier"]
points, msg_list = self.eval_persistency(id_list, data_or_metadata="data")
logger.debug(msg_list)
return (points, msg_list)
@ConfigTerms(term_id="identifier_term")
def rda_f1_02m(self, **kwargs):
"""Indicator RDA-F1-02M: Metadata is identified by a globally unique identifier.
This indicator is linked to the following principle: F1 (meta)data are assigned a globally unique and eternally persistent identifier.
The indicator serves to evaluate whether the identifier of the metadata is globally unique, i.e. that there are no two identical
identifiers that identify different metadata records.
Parameters
----------
identifier_term_data : dict
A dictionary with metadata information about the identifier/s used for the data (see ConfigTerms class for further details)
Returns
-------
points
- 0/100 if the identifier used for the metadata is not globally unique.
- 100/100 if the identifier used for the metadata is globally unique.
msg
Message with the results or recommendations to improve this indicator
"""
id_list = kwargs["Metadata Identifier"]
points, msg_list = self.eval_uniqueness(id_list, data_or_metadata="metadata")
logger.debug(msg_list)
return (points, msg_list)
@ConfigTerms(term_id="identifier_term_data")
def rda_f1_02d(self, **kwargs):
"""Indicator RDA-F1-02D: Data is identified by a globally unique identifier.
This indicator is linked to the following principle: F1 (meta)data are assigned a globally unique and eternally persistent identifier.
The indicator serves to evaluate whether the identifier of the data is globally unique, i.e. that there are no two people that would
use that same identifier for two different digital objects.
Parameters
----------
identifier_term_data : dict
A dictionary with metadata information about the identifier/s used for the data (see ConfigTerms class for further details)
Returns
-------
points
Returns a value (out of 100) that reflects the amount of data identifiers that are globally unique (i.e. DOI, Handle, UUID).
msg
Message with the results or recommendations to improve this indicator
"""
id_list = kwargs["Data Identifier"]
points, msg_list = self.eval_uniqueness(id_list, data_or_metadata="data")
logger.debug(msg_list)
return (points, msg_list)
@ConfigTerms(term_id="terms_findability_richness")
def rda_f2_01m(self, **kwargs):
"""Indicator RDA-F2-01M: Rich metadata is provided to allow discovery.
This indicator is linked to the following principle: F2: Data are described with rich metadata.
The indicator is about the presence of metadata, but also about how much metadata is
provided and how well the provided metadata supports discovery.
Parameters
----------
terms_findability_richness : dict (see ConfigTerms class for further details)
A dictionary with metadata information about the elements that provide findability/discoverability richness.
Returns
-------
points
Returns a value (out of 100) that reflects the grade of compliance with the "Dublin Core Metadata for Resource Discovery".
msg
Message with the results or recommendations to improve this indicator.
"""
terms_findability_dublin_core = ast.literal_eval(
self.config["dublin-core"]["terms_findability_richness"]
)
if not terms_findability_dublin_core:
points, msg_list = (
0,
[
"Dublin Core's terms/elements for 'Metadata for Resource Discovery' are not defined in configuration. Please do so within '[dublin-core]' section."
],
)
else:
dc_term_num = len(terms_findability_dublin_core)
points_per_dc_term = round(100 / dc_term_num)
metadata_keys_not_empty = [k for k, v in kwargs.items() if v]
metadata_keys_not_empty_num = len(metadata_keys_not_empty)
logger.debug(
"Found %s metadata keys with values: %s"
% (metadata_keys_not_empty_num, metadata_keys_not_empty)
)
points = metadata_keys_not_empty_num * points_per_dc_term
msg_list = [
"Found %s (out of %s) metadata elements matching 'Dublin Core Metadata for Resource Discovery' elements"
% (metadata_keys_not_empty_num, dc_term_num)
]
return (points, msg_list)
@ConfigTerms(term_id="identifier_term_data")
def rda_f3_01m(self, **kwargs):
"""Indicator RDA-F3-01M: Metadata includes the identifier for the data.
This indicator is linked to the following principle: F3: Metadata clearly and explicitly include the identifier of the data they describe.
The indicator deals with the inclusion of the reference (i.e. the identifier) of the
digital object in the metadata so that the digital object can be accessed.
Parameters
----------
identifier_term_data : dict
A dictionary with metadata information about the identifier/s used for the data (see ConfigTerms class for further details)
Returns
-------
points
- 100 if metadata contains identifiers for the data.
- 0 otherwise.
msg
Statement about the assessment exercise
"""
id_list = kwargs["Data Identifier"]
msg = "Metadata includes identifier/s for the data: %s" % id_list
points = 100
return (points, [{"message": msg, "points": points}])
def rda_f4_01m(self):
"""Indicator RDA-F4-01M: Metadata is offered in such a way that it can be harvested and indexed.
This indicator is linked to the following principle: F4: (Meta)data are registered or indexed
in a searchable resource.
The indicator tests whether the metadata is offered in such a way that it can be indexed.
In some cases, metadata could be provided together with the data to a local institutional
repository or to a domain-specific or regional portal, or metadata could be included in a
landing page where it can be harvested by a search engine. The indicator remains broad
enough on purpose not to limit the way how and by whom the harvesting and indexing of
the data might be done.
Returns
-------
points
- 100 if metadata could be gathered using any of the supported protocols (OAI-PMH, HTTP).
- 0 otherwise.
msg
Message with the results or recommendations to improve this indicator
"""
if not self.metadata.empty:
msg = "Metadata is gathered programmatically through HTTP (API REST), thus can be harvested and indexed."
points = 100
else:
msg = (
"Could not gather metadata from endpoint: %s. Metadata cannot be harvested and indexed."
% self.api_endpoint
)
points = 0
return (points, [{"message": msg, "points": points}])
@ConfigTerms(term_id="terms_access", validate=True)
def rda_a1_01m(self, only_uri_analysis=False, **kwargs):
"""RDA indicator RDA-A1-01M: Metadata contains information to enable the user to get access to the data.
This indicator is linked to the following principle: A1: (Meta)data are retrievable by their
identifier using a standardised communication protocol. More information about that
principle can be found here.
The indicator refers to the information that is necessary to allow the requester to gain access
to the digital object. It is (i) about whether there are restrictions to access the data (i.e.
access to the data may be open, restricted or closed), (ii) the actions to be taken by a
person who is interested to access the data, in particular when the data has not been
published on the Web and (iii) specifications that the resources are available through
eduGAIN7 or through specialised solutions such as proposed for EPOS.
Returns
-------
points
- The resultant score will be proportional to the percentage of successfully validated metadata elements for data accessibility (see 'terms_access')
msg
Message with the results or recommendations to improve this indicator
"""
points = 0
msg_list = []
term_list = list(kwargs)
msg_intro = (
"Metadata elements related with the user accessibility to data: %s"
% term_list
)
# FIXME: Required to add 'points' even it does not make sense here. Find convention to add intro messages
msg_list.append({"message": msg_intro, "points": 100})
logger.debug(msg_intro)
elements_found = []
for element, element_data in kwargs.items():
element_data_values = element_data.get("values", [])
if not element_data_values:
_msg = "Values for the metadata element '%s' not found" % element
_points = 0
logger.warning(_msg)
else:
_msg = "Found values for the metadata element '%s': %s" % (
element,
element_data_values,
)
_points = 100
elements_found.append(element)
# Report about validation data
element_data_validation = element_data.get("validation", {})
_msg_validation = ""
compliant_vocabularies = []
for vocabulary, validation_data in element_data_validation.items():
valid_data = validation_data.get("valid", [])
if valid_data:
if not _msg_validation:
_msg_validation += ". In addition, values have been successfully validated according to standard vocabulary/ies: %s"
compliant_vocabularies.append(vocabulary)
if compliant_vocabularies:
_msg_validation = _msg_validation % ", ".join(
compliant_vocabularies
)
_msg += _msg_validation
logger.debug(_msg)
msg_list.append({"message": _msg, "points": _points})
# Calculate points
points = len(elements_found) / len(term_list) * 100
return (points, msg_list)
def rda_a1_02m(self):
"""Indicator RDA-A1-02M: Metadata can be accessed manually (i.e. with human intervention).
This indicator is linked to the following principle: A1: (Meta)data are retrievable by their
identifier using a standardised communication protocol.
The indicator refers to any human interactions that are needed if the requester wants to
access metadata. The FAIR principle refers mostly to automated interactions where a
machine is able to access the metadata, but there may also be metadata that require human
interactions. This may be important in cases where the metadata itself contains sensitive
information. Human interaction might involve sending an e-mail to the metadata owner, or
calling by telephone to receive instructions.
Returns
-------
points
100/100 if the link to the manual aquisition of the metadata exists
msg
Message with the results or recommendations to improve this indicator
"""
points = 0
msg = "No reference has been found for the manual obtention of the metadata"
msg_list = []
if self.metadata_access_manual:
if ut.check_link(self.metadata_access_manual[0]):
msg = (
"Documentation for the manual obtention of the metadata can be found in "
+ str(self.metadata_access_manual[0])
)
points = 100
msg_list.append({"message": msg, "points": points})
return (points, msg_list)
def rda_a1_02d(self):
"""Indicator RDA-A1-02D: Data can be accessed manually (i.e. with human intervention).
This indicator is linked to the following principle: A1: (Meta)data are retrievable by their
identifier using a standardised communication protocol.
The indicator refers to any human interactions that are needed if the requester wants to
access data. The FAIR principle refers mostly to automated interactions where a
machine is able to access the metadata, but there may also be metadata that require human
interactions. This may be important in cases where the metadata itself contains sensitive
information. Human interaction might involve sending an e-mail to the metadata owner, or
calling by telephone to receive instructions.
Returns
-------
points
100/100 if the link to the manual aquisition of the data exists
msg
Message with the results or recommendations to improve this indicator
"""
points = 0
msg = "No reference has been found for the manual obtention of the data"
msg_list = []
if self.data_access_manual:
if ut.check_link(self.data_access_manual[0]):
msg = (
"Documentation for the manual obtention of the data can be found in "
+ str(self.data_access_manual[0])
)
points = 100
msg_list.append({"message": msg, "points": points})
return (points, msg_list)
def rda_a1_03m(self):
"""Indicator RDA-A1-03M: Metadata identifier resolves to a metadata record.
This indicator is linked to the following principle: A1: (Meta)data are retrievable by their
identifier using a standardised communication protocol.
This indicator is about the resolution of the metadata identifier. The identifier assigned to
the metadata should be associated with a resolution service that enables access to the
metadata record.
Returns
-------
points
100/100 if the metadata record can be obtained (0/100 otherwise)
msg
Message with the results or recommendations to improve this indicator
"""
points = 0
msg = (
"Metadata record cannot be retrieved from metadata identifier: %s"
% self.item_id
)
if not self.metadata.empty:
points = 100
msg = (
"Metadata record could be retrieved from metadata identifier: %s"
% self.item_id
)
msg_list = [{"message": msg, "points": points}]
return (points, msg_list)
@ConfigTerms(term_id="terms_access")
def rda_a1_03d(self, **kwargs):
"""Indicator RDA-A1-03D: Data identifier resolves to a digital object.
This indicator is linked to the following principle: A1: (Meta)data are retrievable by their
identifier using a standardised communication protocol.
This indicator is about the resolution of the identifier that identifies the digital object. The
identifier assigned to the data should be associated with a formally defined
retrieval/resolution mechanism that enables access to the digital object, or provides access
instructions for access in the case of human-mediated access. The FAIR principle and this
indicator do not say anything about the mutability or immutability of the digital object that
is identified by the data identifier -- this is an aspect that should be governed by a
persistence policy of the data provider.
Returns
-------
points
A number between 0 and 100 to indicate how well this indicator is supported
- 100/100 if all object identifiers are resolvable 0 if none
- For any other case the resultant points will be proportional to the % of resovable identifiers
msg
Message with the results or recommendations to improve this indicator
"""
points = 0
msg = ""
data_id_list = kwargs["Data Identifier"]
data_url_list = kwargs["Download Link"]
if not data_id_list and not data_url_list:
msg = "No URI-based identifier to access the data was found"
logger.warning(msg)
return (
points,
[
{
"message": msg,
"points": points,
}
],
)
# Check if resolvable
data_access_uri = data_id_list + data_url_list
data_access_uri_num = len(data_access_uri)
resolvable_uris = []
for uri in data_access_uri:
resolves = False
schemes = idutils.detect_identifier_schemes(uri)
if not schemes:
logger.warning("Could not get the scheme/s from the value: %s" % uri)
else:
logger.debug(
"Identifier schemes found for the value '%s': %s" % (uri, schemes)
)
if "doi" in schemes or "handle" in schemes:
resolves = ut.resolve_handle(uri)[0]
elif "url" in schemes:
resolves = ut.check_link(uri)
else:
logger.warning(
"Scheme/s used by the identifier not known: %s" % schemes
)
if resolves:
resolvable_uris.append(uri)
resolvable_uris_num = len(resolvable_uris)
if resolvable_uris:
msg = "Found %s/%s resolvable URIs for accessing the data: %s" % (
resolvable_uris_num,
data_access_uri_num,
resolvable_uris,
)
logger.debug(msg)
points = 100
else:
msg = (
"None of the URIs found for accessing the data is resolvable: %s"
% data_access_uri
)
logger.warning(msg)
msg_list = [{"message": msg, "points": points}]
return (points, msg_list)
def rda_a1_04m(self, return_protocol=False):
"""Indicator RDA-A1-04M: Metadata is accessed through standarised protocol.
This indicator is linked to the following principle: A1: (Meta)data are retrievable by their
identifier using a standardised communication protocol.
The indicator concerns the protocol through which the metadata is accessed and requires
the protocol to be defined in a standard.
Returns
-------
points
100/100 if the endpoint protocol is in the accepted list of standarised protocols
msg
Message with the results or recommendations to improve this indicator
"""
points = 0
protocol = ut.get_protocol_scheme(self.api_endpoint)
if protocol in self.terms_access_protocols:
points = 100
msg = "Found a standarised protocol to access the metadata record: " + str(
protocol.upper()
)
else:
msg = (
"Found a non-standarised protocol to access the metadata record: %s"
% str(protocol.upper())
)
msg_list = [{"message": msg, "points": points}]
if return_protocol:
return (points, msg_list, protocol)
return (points, msg_list)
@ConfigTerms(term_id="terms_access")
def rda_a1_04d(self, return_protocol=False, **kwargs):
"""Indicator RDA-A1-04D: Data is accessible through standardized protocol.
This indicator is linked to the following principle: A1: (Meta)data are retrievable by their
identifier using a standardised communication protocol.
The indicator concerns the protocol through which the digital object is accessed and requires
the protocol to be defined in a standard.
Returns
-------
points
100/100 if the download protocol is in the accepted protocols list
msg
Message with the results or recommendations to improve this indicator
"""
points = 0
msg = ""
url = kwargs["Download Link"]
if len(url) == 0:
return (
points,
[
{
"message": "Could not check data access protocol: EPOS metadata element <downloadURL> not found",
"points": points,
}
],
)
protocol_list = []
for link in url:
parsed_endpoint = urllib.parse.urlparse(link)
protocol = parsed_endpoint.scheme
if protocol in self.terms_access_protocols:
points = 100
protocol_list.append(protocol)
if points == 100:
msg = "Found %s standarised protocols to access the data: %s" % (
len(protocol_list),
[protocol.upper() for protocol in protocol_list],
)
msg_list = [{"message": msg, "points": points}]
if return_protocol:
return (points, msg_list, protocol_list)
return (points, msg_list)
@ConfigTerms(term_id="terms_access")
def rda_a1_05d(self, **kwargs):
"""Indicator RDA-A1-05D: Data can be accessed automatically (i.e. by a computer program).
This indicator is linked to the following principle: A1: (Meta)data are retrievable by their
identifier using a standardised communication protocol. More information about that
principle can be found here.
The indicator refers to automated interactions between machines to access digital objects.
The way machines interact and grant access to the digital object will be evaluated by the
indicator.
Returns
-------
points
100/100 if the downloadURL link is provided and resolvable
msg
Message with the results or recommendations to improve this indicator
"""
points = 0
msg_list = []
data_url_list = kwargs["Download Link"]
if data_url_list:
for url in data_url_list:
if ut.check_link(url):
points = 100
msg_list.append(
{
"message": "Data can be accessed programmatically. The provided URL is resolvable: %s"
% str(url),
"points": points,
}
)
else:
return (
points,
[
{
"message": "Could not check data access protocol: EPOS metadata element <downloadURL> not found",
"points": points,
}
],
)
return (points, msg_list)
def rda_a1_1_01m(self):
"""Indicator RDA-A1.1_01M: Metadata is accessible through a free access
protocol.
This indicator is linked to the following principle: A1.1: The protocol is open, free and
universally implementable. More information about that principle can be found here.