diff --git a/sdg/helpers/px.py b/sdg/helpers/px.py index f2d4ef97..ebbb5a5d 100644 --- a/sdg/helpers/px.py +++ b/sdg/helpers/px.py @@ -22,10 +22,13 @@ def keyword(self, k, lang=None): if k not in self.keywords(): raise ValueError(f"'{k}' is not a valid KEYWORD") - if 'TABLE' in self.metadata[k]: - return self.metadata[k]['TABLE'] - else: + metadata_keys = self.metadata[k].keys() + if 'TABLE' not in metadata_keys: + return self.metadata[k] + elif 'TABLE' in metadata_keys and len(metadata_keys) > 1: return self.metadata[k] + else: + return self.metadata[k]['TABLE'] def title(self): @@ -325,7 +328,10 @@ def get_year_column_name(self): def get_units_column_name(self): - return self.keyword('UNITS') + col = self.keyword('UNITS') + if isinstance(col, dict) and 'TABLE' in col: + return col['TABLE'] + return col def get_value_column_name(self): diff --git a/sdg/inputs/InputMetaFiles.py b/sdg/inputs/InputMetaFiles.py index 11f12e03..744775b3 100644 --- a/sdg/inputs/InputMetaFiles.py +++ b/sdg/inputs/InputMetaFiles.py @@ -67,6 +67,7 @@ def add_language_folders(self, meta, filepath): translated_meta = self.read_meta_at_path(translated_filepath) self.apply_metadata_mapping(translated_meta) self.fix_booleans(translated_meta) + self.remove_empty_values(translated_meta) meta[language] = translated_meta @@ -139,3 +140,9 @@ def apply_metadata_mapping(self, metadata): if human_key in metadata and human_key != machine_key: metadata[machine_key] = metadata[human_key] del metadata[human_key] + + + def remove_empty_values(self, meta): + empty_keys = [k for k,v in meta.items() if not v] + for k in empty_keys: + del meta[k] diff --git a/sdg/inputs/InputPxFile.py b/sdg/inputs/InputPxFile.py index d03d7daa..195f9729 100644 --- a/sdg/inputs/InputPxFile.py +++ b/sdg/inputs/InputPxFile.py @@ -89,17 +89,46 @@ def replace_value(value): metadata = {} if not (px.data_has_units() and 'UNITS' in keywords): metadata['computation_units'] = translation_group + '.computation_units' - if 'NOTE' in keywords: - note_value = px.keyword('NOTE') - if isinstance(note_value, str): - metadata['data_footnote'] = translation_group + '.data_footnote' - metadata['page_content'] = translation_group + '.page_content' + if 'NOTEX' in keywords: + notex_value = px.keyword('NOTEX') + notex_header = '' + notex_footer = [] + if isinstance(notex_value, str): + notex_header = translation_group + '.page_content' + elif isinstance(notex_value, dict): + value_keys = notex_value.keys() + for value_key in value_keys: + if value_key == 'TABLE': + notex_header = translation_group + '.page_content' + else: + notex_footer.append(value_key) + if notex_header: + metadata['page_content'] = notex_header + if notex_footer: + if 'footer_fields' not in metadata: + metadata['footer_fields'] = [] + for notex_field in notex_footer: + footer_field = { + "label": translation_group + '.footer_field_label-' + notex_field, + "value": translation_group + '.footer_field_value-' + notex_field + } + metadata['footer_fields'].append(footer_field) if 'INFO' in keywords: metadata['graph_title'] = translation_group + '.graph_title' metadata['indicator_name'] = translation_group + '.indicator_name' for mapped_key in self.meta_map: if mapped_key in keywords: - metadata[self.meta_map[mapped_key]] = translation_group + '.' + mapped_key + mapped_value = px.keyword(mapped_key) + converted_key = self.meta_map[mapped_key] + if isinstance(mapped_value, str): + metadata[converted_key] = translation_group + '.' + mapped_key + elif isinstance(mapped_value, dict): + value_keys = mapped_value.keys() + for value_key in value_keys: + if value_key == 'TABLE': + metadata[converted_key] = translation_group + '.' + mapped_key + else: + metadata[converted_key + '-' + value_key] = translation_group + '.' + mapped_key + '-' + value_key # As a benefit to the Open SGD integration, if the data # is empty, automatically flag it as a non-statistical # indicator. @@ -113,16 +142,16 @@ def replace_value(value): def get_meta_map(self, source): - if source is None: - return {} - elif isinstance(source, dict): - return source - elif isinstance(source, str): + map = {} + if isinstance(source, str): with open(source) as file: - return yaml.load(file, Loader=yaml.FullLoader) + map = yaml.load(file, Loader=yaml.FullLoader) + if isinstance(map, dict): + # Always include NOTE mappeed to itself. + map['NOTE'] = 'NOTE' + return map else: raise Exception("The meta_map parameter is not configured correctly.") - return {} def get_indicator_id_map(self, source): diff --git a/sdg/translations/TranslationInputPx.py b/sdg/translations/TranslationInputPx.py index c68a1212..a5fbc142 100644 --- a/sdg/translations/TranslationInputPx.py +++ b/sdg/translations/TranslationInputPx.py @@ -57,9 +57,6 @@ def execute(self): if has_indicator_options and has_units and translatable_variable == px.get_units_column_name(): renamed_variable = self.indicator_options.get_unit_column() for language in languages: - suffix = '' - if language != default_language: - suffix = '[' + language + ']' translated_variable = px.variable_get_translation_from_value(translatable_variable, language) self.add_translation(language, renamed_variable, renamed_variable, translated_variable) codes = px.codes(translatable_variable) @@ -76,14 +73,35 @@ def execute(self): try: if not (px.data_has_units() and 'UNITS' in px.keywords()): metadata_value = px.keyword('UNITS', language) - self.add_translation(language, translation_group, 'computation_units', metadata_value) + if isinstance(metadata_value, dict) and 'TABLE' in metadata_value: + metadata_value = metadata_value['TABLE'] + if isinstance(metadata_value, str): + self.add_translation(language, translation_group, 'computation_units', metadata_value) except: pass try: - metadata_value = px.keyword('NOTE', language) - if isinstance(metadata_value, str): - self.add_translation(language, translation_group, 'data_footnote', metadata_value) - self.add_translation(language, translation_group, 'page_content', metadata_value) + if 'NOTEX' in px.keywords(): + metadata_value = px.keyword('NOTEX', language) + notex_header = '' + notex_footer = [] + if isinstance(metadata_value, str): + notex_header = metadata_value + elif isinstance(metadata_value, dict): + untranslated_value = px.keyword('NOTEX') + value_keys = untranslated_value.keys() + for value_key in value_keys: + if value_key == 'TABLE': + notex_header = metadata_value['TABLE'] + else: + notex_footer.append(value_key) + if notex_header: + self.add_translation(language, translation_group, 'page_content', notex_header) + if notex_footer: + for notex_field in notex_footer: + untranslated_variable = notex_field + translated_variable = px.variable_get_translation_from_value(untranslated_variable, language) + self.add_translation(language, translation_group, 'footer_field_label-' + untranslated_variable, translated_variable) + self.add_translation(language, translation_group, 'footer_field_value-' + untranslated_variable, metadata_value[translated_variable]) except: pass try: @@ -95,21 +113,35 @@ def execute(self): for mapped_key in self.meta_map: try: metadata_value = px.keyword(mapped_key, language) - self.add_translation(language, translation_group, mapped_key, metadata_value) - except: + if isinstance(metadata_value, str): + self.add_translation(language, translation_group, mapped_key, metadata_value) + elif isinstance(metadata_value, dict): + untranslated_value = px.keyword(mapped_key) + value_keys = untranslated_value.keys() + for value_key in value_keys: + if value_key == 'TABLE': + self.add_translation(language, translation_group, mapped_key, metadata_value['TABLE']) + else: + # We assume this is a variable. + untranslated_variable = value_key + translated_variable = px.variable_get_translation_from_value(untranslated_variable, language) + self.add_translation(language, translation_group, mapped_key + '-' + value_key, metadata_value[translated_variable]) + except Exception as e: + print(e) pass def get_meta_map(self, source): - if source is None: - return {} - elif isinstance(source, dict): - return source - elif isinstance(source, str): + map = {} + if isinstance(source, str): with open(source) as file: - return yaml.load(file, Loader=yaml.FullLoader) + map = yaml.load(file, Loader=yaml.FullLoader) + if isinstance(map, dict): + # Always include NOTE mappeed to itself. + map['NOTE'] = 'NOTE' + return map else: raise Exception("The meta_map parameter is not configured correctly.") - return {} + def get_indicator_id_map(self, source): if isinstance(source, dict):