Remote data and metadata files

sdg/inputs/InputCsvDataRemote.py

@@ -0,0 +1,49 @@
+import pandas as pd
+from sdg.inputs import InputBase
+import yaml
+
+class InputCsvDataRemote(InputBase):
+    """Sources of SDG data that are remote CSV files."""
+
+    def __init__(self,
+        indicator_id_map=None,
+        logging=None,
+        column_map=None,
+        code_map=None,
+        dtype=None,
+        request_params=None,
+    ):
+        """Constructor for InputCsvDataRemote.
+
+        Keyword arguments:
+        indicator_id_map: A dict mapping sources (remote URLs) to
+        lists of indicator ids.
+        """
+        InputBase.__init__(self,
+            logging=logging,
+            column_map=column_map,
+            code_map=code_map,
+            request_params=request_params,
+        )
+        self.indicator_id_map = self.get_indicator_id_map(indicator_id_map)
+        self.dtype = {} if dtype is None else dtype
+
+
+    def execute(self, indicator_options):
+        for source, indicator_ids in self.indicator_id_map.items():
+            data = pd.read_csv(source, dtype=self.dtype)
+            if not isinstance(indicator_ids, list):
+                indicator_ids = [indicator_ids]
+            for inid in indicator_ids:
+                self.add_indicator(inid, data=data, options=indicator_options)
+
+
+    def get_indicator_id_map(self, source):
+        if isinstance(source, dict):
+            return source
+        elif isinstance(source, str):
+            with open(source) as file:
+                return yaml.load(file, Loader=yaml.FullLoader)
+        else:
+            raise Exception("The indicator_id_map parameter is not configured correctly.")
+        return {}

sdg/inputs/InputYamlMetaRemote.py

@@ -0,0 +1,48 @@
+import pandas as pd
+from sdg.inputs import InputBase
+import yaml
+
+class InputYamlMetaRemote(InputBase):
+    """Sources of SDG metadata that are remote YAML files."""
+
+    def __init__(self,
+        indicator_id_map=None,
+        logging=None,
+        metadata_mapping=None,
+        request_params=None,
+    ):
+        """Constructor for InputCsvDataRemote.
+
+        Keyword arguments:
+        indicator_id_map: A dict mapping sources (remote URLs) to
+        lists of indicator ids.
+        metadata_mapping: A dict mapping human-readable labels to machine keys
+          or a path to a CSV file
+        """
+        InputBase.__init__(self,
+            logging=logging,
+            request_params=request_params,
+        )
+        self.indicator_id_map = self.get_indicator_id_map(indicator_id_map)
+        self.metadata_mapping = metadata_mapping
+
+
+    def execute(self, indicator_options):
+        for source, indicator_ids in self.indicator_id_map.items():
+            file_contents = self.fetch_file(source)
+            meta = yaml.safe_load(file_contents)
+            if not isinstance(indicator_ids, list):
+                indicator_ids = [indicator_ids]
+            for inid in indicator_ids:
+                self.add_indicator(inid, meta=meta, options=indicator_options)
+
+
+    def get_indicator_id_map(self, source):
+        if isinstance(source, dict):
+            return source
+        elif isinstance(source, str):
+            with open(source) as file:
+                return yaml.load(file, Loader=yaml.FullLoader)
+        else:
+            raise Exception("The indicator_id_map parameter is not configured correctly.")
+        return {}

sdg/inputs/__init__.py

@@ -20,3 +20,5 @@
 from .InputSdmxMl_UnitedNationsApi import InputSdmxMl_UnitedNationsApi
 from .InputSdgMetadata import InputSdgMetadata
 from .InputPxFile import InputPxFile
+from .InputCsvDataRemote import InputCsvDataRemote
+from .InputYamlMetaRemote import InputYamlMetaRemote

sdg/open_sdg.py

@@ -540,6 +540,8 @@ def open_sdg_input_from_dict(params, options):
         'InputWordMeta',
         'InputSdgMetadata',
         'InputPxFile',
+        'InputCsvDataRemote',
+        'InputYamlMetaRemote',
     ]
     if input_class not in allowed:
         raise KeyError("Input class '%s' is not one of: %s." % (input_class, ', '.join(allowed)))
@@ -588,6 +590,10 @@ def open_sdg_input_from_dict(params, options):
         input_instance = sdg.inputs.InputSdgMetadata(**params)
     elif input_class == 'InputPxFile':
         input_instance = sdg.inputs.InputPxFile(**params)
+    elif input_class == 'InputCsvDataRemote':
+        input_instance = sdg.inputs.InputCsvDataRemote(**params)
+    elif input_class == 'InputYamlMetaRemote':
+        input_instance =  sdg.inputs.InputYamlMetaRemote(**params)
 
     return input_instance