Skip to content

PX Web - support for multi-value tags #381

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
brockfanning wants to merge 6 commits into
base: 2.5.0-dev
Choose a base branch
from
Open

PX Web - support for multi-value tags #381

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c70a214
Progress
brockfanning Feb 24, 2026
6d74b75
Remove empty values from translated metadata to avoid problems
brockfanning Feb 26, 2026
d88f23d
Progress
brockfanning Feb 26, 2026
8eb5a26
Hardcode NOTE support.
brockfanning Mar 6, 2026
c96def2
Handle multi-variable unit tag.
brockfanning Mar 26, 2026
0dccadd
Handle units with multi-values.
brockfanning Apr 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions sdg/helpers/px.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,13 @@ def keyword(self, k, lang=None):
if k not in self.keywords():
raise ValueError(f"'{k}' is not a valid KEYWORD")

if 'TABLE' in self.metadata[k]:
return self.metadata[k]['TABLE']
else:
metadata_keys = self.metadata[k].keys()
if 'TABLE' not in metadata_keys:
return self.metadata[k]
elif 'TABLE' in metadata_keys and len(metadata_keys) > 1:
return self.metadata[k]
else:
return self.metadata[k]['TABLE']


def title(self):
Expand Down Expand Up @@ -325,7 +328,10 @@ def get_year_column_name(self):


def get_units_column_name(self):
return self.keyword('UNITS')
col = self.keyword('UNITS')
if isinstance(col, dict) and 'TABLE' in col:
return col['TABLE']
return col


def get_value_column_name(self):
Expand Down
7 changes: 7 additions & 0 deletions sdg/inputs/InputMetaFiles.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ def add_language_folders(self, meta, filepath):
translated_meta = self.read_meta_at_path(translated_filepath)
self.apply_metadata_mapping(translated_meta)
self.fix_booleans(translated_meta)
self.remove_empty_values(translated_meta)
meta[language] = translated_meta


Expand Down Expand Up @@ -139,3 +140,9 @@ def apply_metadata_mapping(self, metadata):
if human_key in metadata and human_key != machine_key:
metadata[machine_key] = metadata[human_key]
del metadata[human_key]


def remove_empty_values(self, meta):
empty_keys = [k for k,v in meta.items() if not v]
for k in empty_keys:
del meta[k]
55 changes: 42 additions & 13 deletions sdg/inputs/InputPxFile.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,17 +89,46 @@ def replace_value(value):
metadata = {}
if not (px.data_has_units() and 'UNITS' in keywords):
metadata['computation_units'] = translation_group + '.computation_units'
if 'NOTE' in keywords:
note_value = px.keyword('NOTE')
if isinstance(note_value, str):
metadata['data_footnote'] = translation_group + '.data_footnote'
metadata['page_content'] = translation_group + '.page_content'
if 'NOTEX' in keywords:
notex_value = px.keyword('NOTEX')
notex_header = ''
notex_footer = []
if isinstance(notex_value, str):
notex_header = translation_group + '.page_content'
elif isinstance(notex_value, dict):
value_keys = notex_value.keys()
for value_key in value_keys:
if value_key == 'TABLE':
notex_header = translation_group + '.page_content'
else:
notex_footer.append(value_key)
if notex_header:
metadata['page_content'] = notex_header
if notex_footer:
if 'footer_fields' not in metadata:
metadata['footer_fields'] = []
for notex_field in notex_footer:
footer_field = {
"label": translation_group + '.footer_field_label-' + notex_field,
"value": translation_group + '.footer_field_value-' + notex_field
}
metadata['footer_fields'].append(footer_field)
if 'INFO' in keywords:
metadata['graph_title'] = translation_group + '.graph_title'
metadata['indicator_name'] = translation_group + '.indicator_name'
for mapped_key in self.meta_map:
if mapped_key in keywords:
metadata[self.meta_map[mapped_key]] = translation_group + '.' + mapped_key
mapped_value = px.keyword(mapped_key)
converted_key = self.meta_map[mapped_key]
if isinstance(mapped_value, str):
metadata[converted_key] = translation_group + '.' + mapped_key
elif isinstance(mapped_value, dict):
value_keys = mapped_value.keys()
for value_key in value_keys:
if value_key == 'TABLE':
metadata[converted_key] = translation_group + '.' + mapped_key
else:
metadata[converted_key + '-' + value_key] = translation_group + '.' + mapped_key + '-' + value_key
# As a benefit to the Open SGD integration, if the data
# is empty, automatically flag it as a non-statistical
# indicator.
Expand All @@ -113,16 +142,16 @@ def replace_value(value):


def get_meta_map(self, source):
if source is None:
return {}
elif isinstance(source, dict):
return source
elif isinstance(source, str):
map = {}
if isinstance(source, str):
with open(source) as file:
return yaml.load(file, Loader=yaml.FullLoader)
map = yaml.load(file, Loader=yaml.FullLoader)
if isinstance(map, dict):
# Always include NOTE mappeed to itself.
map['NOTE'] = 'NOTE'
return map
else:
raise Exception("The meta_map parameter is not configured correctly.")
return {}


def get_indicator_id_map(self, source):
Expand Down
66 changes: 49 additions & 17 deletions sdg/translations/TranslationInputPx.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,6 @@ def execute(self):
if has_indicator_options and has_units and translatable_variable == px.get_units_column_name():
renamed_variable = self.indicator_options.get_unit_column()
for language in languages:
suffix = ''
if language != default_language:
suffix = '[' + language + ']'
translated_variable = px.variable_get_translation_from_value(translatable_variable, language)
self.add_translation(language, renamed_variable, renamed_variable, translated_variable)
codes = px.codes(translatable_variable)
Expand All @@ -76,14 +73,35 @@ def execute(self):
try:
if not (px.data_has_units() and 'UNITS' in px.keywords()):
metadata_value = px.keyword('UNITS', language)
self.add_translation(language, translation_group, 'computation_units', metadata_value)
if isinstance(metadata_value, dict) and 'TABLE' in metadata_value:
metadata_value = metadata_value['TABLE']
if isinstance(metadata_value, str):
self.add_translation(language, translation_group, 'computation_units', metadata_value)
except:
pass
try:
metadata_value = px.keyword('NOTE', language)
if isinstance(metadata_value, str):
self.add_translation(language, translation_group, 'data_footnote', metadata_value)
self.add_translation(language, translation_group, 'page_content', metadata_value)
if 'NOTEX' in px.keywords():
metadata_value = px.keyword('NOTEX', language)
notex_header = ''
notex_footer = []
if isinstance(metadata_value, str):
notex_header = metadata_value
elif isinstance(metadata_value, dict):
untranslated_value = px.keyword('NOTEX')
value_keys = untranslated_value.keys()
for value_key in value_keys:
if value_key == 'TABLE':
notex_header = metadata_value['TABLE']
else:
notex_footer.append(value_key)
if notex_header:
self.add_translation(language, translation_group, 'page_content', notex_header)
if notex_footer:
for notex_field in notex_footer:
untranslated_variable = notex_field
translated_variable = px.variable_get_translation_from_value(untranslated_variable, language)
self.add_translation(language, translation_group, 'footer_field_label-' + untranslated_variable, translated_variable)
self.add_translation(language, translation_group, 'footer_field_value-' + untranslated_variable, metadata_value[translated_variable])
except:
pass
try:
Expand All @@ -95,21 +113,35 @@ def execute(self):
for mapped_key in self.meta_map:
try:
metadata_value = px.keyword(mapped_key, language)
self.add_translation(language, translation_group, mapped_key, metadata_value)
except:
if isinstance(metadata_value, str):
self.add_translation(language, translation_group, mapped_key, metadata_value)
elif isinstance(metadata_value, dict):
untranslated_value = px.keyword(mapped_key)
value_keys = untranslated_value.keys()
for value_key in value_keys:
if value_key == 'TABLE':
self.add_translation(language, translation_group, mapped_key, metadata_value['TABLE'])
else:
# We assume this is a variable.
untranslated_variable = value_key
translated_variable = px.variable_get_translation_from_value(untranslated_variable, language)
self.add_translation(language, translation_group, mapped_key + '-' + value_key, metadata_value[translated_variable])
except Exception as e:
print(e)
pass

def get_meta_map(self, source):
if source is None:
return {}
elif isinstance(source, dict):
return source
elif isinstance(source, str):
map = {}
if isinstance(source, str):
with open(source) as file:
return yaml.load(file, Loader=yaml.FullLoader)
map = yaml.load(file, Loader=yaml.FullLoader)
if isinstance(map, dict):
# Always include NOTE mappeed to itself.
map['NOTE'] = 'NOTE'
return map
else:
raise Exception("The meta_map parameter is not configured correctly.")
return {}


def get_indicator_id_map(self, source):
if isinstance(source, dict):
Expand Down
Loading