diff --git a/src/swell/configuration/jedi/bufr2ioda/bufr_ncep_1bamua_n15.yaml b/src/swell/configuration/jedi/bufr2ioda/bufr2ioda_x/bufr_ncep_1bamua_n15.yaml similarity index 100% rename from src/swell/configuration/jedi/bufr2ioda/bufr_ncep_1bamua_n15.yaml rename to src/swell/configuration/jedi/bufr2ioda/bufr2ioda_x/bufr_ncep_1bamua_n15.yaml diff --git a/src/swell/configuration/jedi/bufr2ioda/bufr_ncep_1bamua_ta.yaml b/src/swell/configuration/jedi/bufr2ioda/bufr2ioda_x/bufr_ncep_1bamua_ta.yaml similarity index 100% rename from src/swell/configuration/jedi/bufr2ioda/bufr_ncep_1bamua_ta.yaml rename to src/swell/configuration/jedi/bufr2ioda/bufr2ioda_x/bufr_ncep_1bamua_ta.yaml diff --git a/src/swell/configuration/jedi/bufr2ioda/bufr_ncep_1bmhs.yaml b/src/swell/configuration/jedi/bufr2ioda/bufr2ioda_x/bufr_ncep_1bmhs.yaml similarity index 100% rename from src/swell/configuration/jedi/bufr2ioda/bufr_ncep_1bmhs.yaml rename to src/swell/configuration/jedi/bufr2ioda/bufr2ioda_x/bufr_ncep_1bmhs.yaml diff --git a/src/swell/configuration/jedi/bufr2ioda/bufr_ncep_atms.yaml b/src/swell/configuration/jedi/bufr2ioda/bufr2ioda_x/bufr_ncep_atms.yaml similarity index 100% rename from src/swell/configuration/jedi/bufr2ioda/bufr_ncep_atms.yaml rename to src/swell/configuration/jedi/bufr2ioda/bufr2ioda_x/bufr_ncep_atms.yaml diff --git a/src/swell/configuration/jedi/bufr2ioda/bufr_ncep_atms_remap.yaml b/src/swell/configuration/jedi/bufr2ioda/bufr2ioda_x/bufr_ncep_atms_remap.yaml similarity index 100% rename from src/swell/configuration/jedi/bufr2ioda/bufr_ncep_atms_remap.yaml rename to src/swell/configuration/jedi/bufr2ioda/bufr2ioda_x/bufr_ncep_atms_remap.yaml diff --git a/src/swell/configuration/jedi/bufr2ioda/bufr_ncep_atms_ta_remap.yaml b/src/swell/configuration/jedi/bufr2ioda/bufr2ioda_x/bufr_ncep_atms_ta_remap.yaml similarity index 100% rename from src/swell/configuration/jedi/bufr2ioda/bufr_ncep_atms_ta_remap.yaml rename to src/swell/configuration/jedi/bufr2ioda/bufr2ioda_x/bufr_ncep_atms_ta_remap.yaml diff --git a/src/swell/configuration/jedi/bufr2ioda/bufr_ncep_mtiasi.yaml b/src/swell/configuration/jedi/bufr2ioda/bufr2ioda_x/bufr_ncep_mtiasi.yaml similarity index 100% rename from src/swell/configuration/jedi/bufr2ioda/bufr_ncep_mtiasi.yaml rename to src/swell/configuration/jedi/bufr2ioda/bufr2ioda_x/bufr_ncep_mtiasi.yaml diff --git a/src/swell/configuration/jedi/bufr2ioda/bufr_ncep_prepbufr_aircft.yaml b/src/swell/configuration/jedi/bufr2ioda/bufr2ioda_x/bufr_ncep_prepbufr_aircft.yaml similarity index 100% rename from src/swell/configuration/jedi/bufr2ioda/bufr_ncep_prepbufr_aircft.yaml rename to src/swell/configuration/jedi/bufr2ioda/bufr2ioda_x/bufr_ncep_prepbufr_aircft.yaml diff --git a/src/swell/configuration/jedi/bufr2ioda/bufr_ncep_satwind_avhrr.yaml b/src/swell/configuration/jedi/bufr2ioda/bufr2ioda_x/bufr_ncep_satwind_avhrr.yaml similarity index 100% rename from src/swell/configuration/jedi/bufr2ioda/bufr_ncep_satwind_avhrr.yaml rename to src/swell/configuration/jedi/bufr2ioda/bufr2ioda_x/bufr_ncep_satwind_avhrr.yaml diff --git a/src/swell/configuration/jedi/bufr2ioda/bufr_ncep_sevcsr.yaml b/src/swell/configuration/jedi/bufr2ioda/bufr2ioda_x/bufr_ncep_sevcsr.yaml similarity index 100% rename from src/swell/configuration/jedi/bufr2ioda/bufr_ncep_sevcsr.yaml rename to src/swell/configuration/jedi/bufr2ioda/bufr2ioda_x/bufr_ncep_sevcsr.yaml diff --git a/src/swell/configuration/jedi/bufr2ioda/bufr2netcdf_x/bufr_mapping_1bamua_ta.yaml b/src/swell/configuration/jedi/bufr2ioda/bufr2netcdf_x/bufr_mapping_1bamua_ta.yaml new file mode 100644 index 000000000..514e807e6 --- /dev/null +++ b/src/swell/configuration/jedi/bufr2ioda/bufr2netcdf_x/bufr_mapping_1bamua_ta.yaml @@ -0,0 +1,50 @@ +bufr: + variables: + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + second: "*/SECO" + latitude: + query: "*/CLAT" + longitude: + query: "*/CLON" + brightnessTemp: + query: "*/BRITCSTC/TMBR" + +encoder: + dimensions: + - name: Channel + paths: + - "*/BRITCSTC" + + variables: + - name: "MetaData/dateTime" + source: variables/timestamp + longName: "dateTime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degrees_north" + range: [-90, 90] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degrees_east" + range: [-180, 180] + + # ObsValue + # For now, store the antennaTemperature as brightnessTemperature + - name: "ObsValue/brightnessTemperature" + source: variables/antennaTemperature + longName: "Antenna Temperature" + units: "K" + range: [90, 380] + chunks: [1000, 15] + compressionLevel: 4 diff --git a/src/swell/configuration/jedi/bufr2ioda/bufr2netcdf_x/bufr_mapping_1bmhs.yaml b/src/swell/configuration/jedi/bufr2ioda/bufr2netcdf_x/bufr_mapping_1bmhs.yaml new file mode 100644 index 000000000..72a5ac0f5 --- /dev/null +++ b/src/swell/configuration/jedi/bufr2ioda/bufr2netcdf_x/bufr_mapping_1bmhs.yaml @@ -0,0 +1,47 @@ +bufr: + variables: + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + second: "*/SECO" + latitude: + query: "*/CLAT" + longitude: + query: "*/CLON" + brightnessTemp: + query: "*/BRITCSTC/TMBR" + +encoder: + dimensions: + - name: Channel + paths: + - "*/BRITCSTC" + + variables: + - name: "MetaData/dateTime" + source: variables/timestamp + longName: "dateTime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degrees_north" + range: [-90, 90] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degrees_east" + range: [-180, 180] + + - name: "ObsValue/brightnessTemperature" + coordinates: "longitude latitude Channel" + source: variables/brightnessTemp + longName: "Brightness Temperature" + units: "K" + diff --git a/src/swell/configuration/jedi/bufr2ioda/bufr2netcdf_x/bufr_mapping_mhs.yaml b/src/swell/configuration/jedi/bufr2ioda/bufr2netcdf_x/bufr_mapping_mhs.yaml new file mode 100644 index 000000000..72a5ac0f5 --- /dev/null +++ b/src/swell/configuration/jedi/bufr2ioda/bufr2netcdf_x/bufr_mapping_mhs.yaml @@ -0,0 +1,47 @@ +bufr: + variables: + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + second: "*/SECO" + latitude: + query: "*/CLAT" + longitude: + query: "*/CLON" + brightnessTemp: + query: "*/BRITCSTC/TMBR" + +encoder: + dimensions: + - name: Channel + paths: + - "*/BRITCSTC" + + variables: + - name: "MetaData/dateTime" + source: variables/timestamp + longName: "dateTime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degrees_north" + range: [-90, 90] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degrees_east" + range: [-180, 180] + + - name: "ObsValue/brightnessTemperature" + coordinates: "longitude latitude Channel" + source: variables/brightnessTemp + longName: "Brightness Temperature" + units: "K" + diff --git a/src/swell/configuration/jedi/bufr2ioda/bufr2netcdf_x/bufr_mapping_mtiasi.yaml b/src/swell/configuration/jedi/bufr2ioda/bufr2netcdf_x/bufr_mapping_mtiasi.yaml new file mode 100644 index 000000000..25f6d667e --- /dev/null +++ b/src/swell/configuration/jedi/bufr2ioda/bufr2netcdf_x/bufr_mapping_mtiasi.yaml @@ -0,0 +1,55 @@ +bufr: + variables: + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + second: "*/SECO" + latitude: + query: "*/CLAT" + longitude: + query: "*/CLON" + brightnessTemp: + query: "*/BRITCSTC/TMBR" + +encoder: + + dimensions: + - name: Channel + source: variables/sensorChannelNumber + path: "*/IASICHN" + + - name: Cluster + path: "*/IASIL1CS" + + - name: Band + path: "*/IASIL1CB" + + variables: + - name: "MetaData/dateTime" + source: variables/timestamp + longName: "dateTime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degrees_north" + range: [-90, 90] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degrees_east" + range: [-180, 180] + +# The unit from BUFR is W m-2 sr-1 m -- this is radiance per wavenumber +# - name: "ObsValue/spectralRadiance" + - name: "ObsValue/radiance" + source: variables/spectralRadiance + longName: "IASI Spectral Radiance" + units: "W m-2 sr-1" + diff --git a/src/swell/suites/convert_bufr/suite_config.py b/src/swell/suites/convert_bufr/suite_config.py index ce1e0f4cc..91ea01562 100644 --- a/src/swell/suites/convert_bufr/suite_config.py +++ b/src/swell/suites/convert_bufr/suite_config.py @@ -37,6 +37,7 @@ class SuiteConfig(QuestionContainer, Enum): "gsi_bcs/*.txt", ]), qd.bufr_obs_classes([ + "ncep_mhs_bufr", "ncep_1bamua_bufr", "ncep_mtiasi_bufr", ]), diff --git a/src/swell/tasks/bufr_to_ioda.py b/src/swell/tasks/bufr_to_ioda.py index 7e8cdd7db..665211804 100644 --- a/src/swell/tasks/bufr_to_ioda.py +++ b/src/swell/tasks/bufr_to_ioda.py @@ -20,18 +20,26 @@ # Dictionary linking each obs type to the appropriate yaml template bufr2ioda_obs_type_dict = { - '1bmhs': 'bufr_ncep_1bmhs.yaml', - 'amsua': 'bufr_ncep_1bamua_ta.yaml', - '1bamua': 'bufr_ncep_1bamua_ta.yaml', - 'ncep_1bamua_bufr': 'bufr_ncep_1bamua_ta.yaml', - 'atms': 'bufr_ncep_atms.yaml', - 'mtiasi': 'bufr_ncep_mtiasi.yaml', - 'ncep_mtiasi_bufr': 'bufr_ncep_mtiasi.yaml', - 'satwind': 'bufr_ncep_satwind_avhrr.yaml', - 'aircft': 'bufr_ncep_prepbufr_aircft.yaml', - 'sevcsr': 'bufr_ncep_sevcsr.yaml' + 'ncep_1bamua_bufr': 'bufr_mapping_1bamua_ta.yaml', + '1bamua': 'bufr_mapping_1bamua_ta.yaml', + 'ncep_mhs_bufr': 'bufr_mapping_mhs.yaml', + '1bmhs': 'bufr_mapping_mhs.yaml', + 'mhs': 'bufr_mapping_mhs.yaml', + 'ncep_mtiasi_bufr': 'bufr_mapping_mtiasi.yaml', + 'mtiasi': 'bufr_mapping_mtiasi.yaml', } + # '1bmhs': 'bufr_mapping_1bmhs.yaml', + # 'ncep_1bmhs_bufr': 'bufr_mapping_1bmhs.yaml', + # 'amsua': 'bufr_mapping_1bamua_ta.yaml', + # '1bamua': 'bufr_mapping_1bamua_ta.yaml', + # 'ncep_1bamua_bufr': 'bufr_mapping_1bamua_ta.yaml', + # 'atms': 'bufr_mapping_atms.yaml', + # # 'mtiasi': 'bufr_mapping_mtiasi.yaml', + # 'ncep_mtiasi_bufr': 'bufr_mapping_mtiasi.yaml', + # 'satwind': 'bufr_mapping_satwind_avhrr.yaml', + # 'aircft': 'bufr_mapping_prepbufr_aircft.yaml', + # 'sevcsr': 'bufr_mapping_sevcsr.yaml' # -------------------------------------------------------------------------------------------------- @@ -45,19 +53,23 @@ def find_obstype_match(self, filename): Dictionary: bufr2ioda_obs_type_dict = { - '1bmhs': 'bufr_ncep_1bmhs.yaml', - 'amsua': 'bufr_ncep_1bamua_ta.yaml', - '1bamua': 'bufr_ncep_1bamua_ta.yaml', - 'atms': 'bufr_ncep_atms.yaml', - 'mtiasi': 'bufr_ncep_mtiasi.yaml', - 'satwind': 'bufr_ncep_satwind_avhrr.yaml', - 'aircft': 'bufr_ncep_prepbufr_aircft.yaml', - 'sevcsr': 'bufr_ncep_sevcsr.yaml' + '1bmhs': 'bufr_mapping_1bmhs.yaml', + 'amsua': 'bufr_mapping_1bamua_ta.yaml', + '1bamua': 'bufr_mapping_1bamua_ta.yaml', + 'atms': 'bufr_mapping_atms.yaml', + 'mtiasi': 'bufr_mapping_mtiasi.yaml', + 'satwind': 'bufr_mapping_satwind_avhrr.yaml', + 'aircft': 'bufr_mapping_prepbufr_aircft.yaml', + 'sevcsr': 'bufr_mapping_sevcsr.yaml' } """ + print(f"filename: {filename}") parts = filename.split('.') + self.logger.info(f"parts: {parts}") + print(f"parts: {parts}") + for part in parts: if part in bufr2ioda_obs_type_dict: self.logger.info(f"Match found: {part}") @@ -67,7 +79,7 @@ def find_obstype_match(self, filename): # -------------------------------------------------------------------------------------------------- - def generate_iodaconv_yaml(self, + def get_bufr_mapping_yaml(self, bufr_file_source_path, ioda_file_target_path, path_to_ioda_conv_yaml_tmpl_dir, @@ -80,59 +92,67 @@ def generate_iodaconv_yaml(self, yaml_file_source: yaml file to use to replicate the structure for the specific obs_type yaml_file_target: ''' - # Value to insert into the yaml file as the value for 'obsdatain'. - # Source file ~ bufr file to be converted - obsdatain = bufr_file_source_path - - # Value to insert into the yaml file as the value for 'obsdataout'. - # Target file ~ conversion output file name - obsdataout = ioda_file_target_path - # find the obs type from file name - bufr_file_obs_type = self.find_obstype_match(obsdatain) + # Copy the yaml file + try: + # Value to insert into the yaml file as the value for 'obsdatain'. + # Source file ~ bufr file to be converted + obsdatain = bufr_file_source_path - # ----------------------------------------------------------------------------------------- + # Value to insert into the yaml file as the value for 'obsdataout'. + # Target file ~ conversion output file name + obsdataout = ioda_file_target_path - if yaml_file_source is None: - # Path to use as the yaml template - yaml_file_source = os.path.join(path_to_ioda_conv_yaml_tmpl_dir, - f'{bufr2ioda_obs_type_dict[bufr_file_obs_type]}') + # find the obs type from file name + bufr_file_obs_type = self.find_obstype_match(obsdatain) + except FileNotFoundError: + self.logger.info(f'bufr_file_source_path: {bufr_file_source_path} ------------------------------ ') + self.logger.info(f'obsdatain: {obsdatain} ------------------------------ ') + self.logger.info(f'obsdataout: {obsdataout} ------------------------------ ') + self.logger.info(f'ioda_file_target_path: {ioda_file_target_path} ------------------------------ ') + self.logger.info(f'bufr_file_obs_type: {bufr_file_obs_type} ------------------------------ ') + except yaml.YAMLError as e: + self.logger.info(f'Error processing find_obstype_match: {e}') # Determine the target path of the generated yaml file # ---------------------------------------------------- if yaml_file_target is None: # Overwrite original if no output file is specified - yaml_file_target = os.path.join(self.cycle_dir(), 'generated_iodaconv_input.yaml') + yaml_file_target = os.path.join(self.cycle_dir(), f'bufr_mapping_{bufr_file_obs_type}.yaml') - self.logger.info(f'YAML template used: {yaml_file_source}.------------------------------ ') - self.logger.info(f'YAML file saved as {yaml_file_target}. ------------------------------ ') + self.logger.info(f'YAML template used: {self.cycle_dir()}.------------------------------ ') + self.logger.info(f'yaml_file_source: {yaml_file_source}.------------------------------ ') + self.logger.info(f'yaml_file_target {yaml_file_target}. ------------------------------ ') - # Copy the yaml file - subprocess.run(['cp', yaml_file_source, yaml_file_target]) - - # Generate the yaml file that will be used in the conversion step ~ bufr2ioda.x [yaml file] # ----------------------------------------------------------------------------------------- - try: - # Load the YAML template file - with open(yaml_file_source, 'r') as file: - yaml_str = file.read() - # Construct dictionary to fill yaml file - template_dictionary = {'obsdatain': obsdatain, - 'obsdataout': obsdataout} - # Apply the replacements for input and output file paths - yaml_str = template_string_jinja2(self.logger, - templated_string=yaml_str, - dictionary_of_templates=template_dictionary) + # Copy the yaml file + try: + if yaml_file_source is None: + # Path to use as the yaml template + yaml_file_source = os.path.join(path_to_ioda_conv_yaml_tmpl_dir, + f'{bufr2ioda_obs_type_dict[bufr_file_obs_type]}') + except FileNotFoundError: + self.logger.info(f'Error: File "{yaml_file_source}" not found.') + self.logger.info(f'Error: "{bufr2ioda_obs_type_dict}" not found.') + self.logger.info(f'Error: "{path_to_ioda_conv_yaml_tmpl_dir}" not found.') + self.logger.info(f'Error: "{bufr_file_obs_type}" not found.') + except yaml.YAMLError as e: + self.logger.info(f'Error processing YAML file: {e}') - # Load the dictionary - yaml_content = yaml.safe_load(yaml_str) + # Determine the target path of the generated yaml file + # ---------------------------------------------------- + if yaml_file_target is None: + # Overwrite original if no output file is specified + yaml_file_target = os.path.join(self.cycle_dir(), f'bufr_mapping_{bufr_file_obs_type}.yaml') - with open(yaml_file_target, 'w') as file: - yaml.dump(yaml_content, file, default_flow_style=False, sort_keys=False) - self.logger.info(f'Updated YAML file content: {yaml_file_target}') + self.logger.info(f'YAML template used: {yaml_file_source}.------------------------------ ') + self.logger.info(f'YAML file saved as {yaml_file_target}. ------------------------------ ') + # Copy the yaml file + try: + subprocess.run(['cp', yaml_file_source, yaml_file_target]) except FileNotFoundError: self.logger.info(f'Error: File "{yaml_file_source}" not found.') except yaml.YAMLError as e: @@ -153,7 +173,8 @@ def execute(self) -> None: # Set the Bufr2Ioda Yaml Template Directory path_to_ioda_conv_yaml_tmpl_dir = os.path.join(self.experiment_path(), - 'configuration/jedi/bufr2ioda') + 'configuration/jedi/bufr2ioda/bufr2netcdf_x/') + self.logger.info(f'Path to yaml files found: {path_to_ioda_conv_yaml_tmpl_dir}') # Get list of all files in cycle dir with .bufr_d suffix or *bufr* bufr_path_files_pattern = os.path.join(bufr_dir, '*bufr*') @@ -173,31 +194,41 @@ def execute(self) -> None: # Source file ~ bufr file to be converted bufr_file_source_path = os.path.basename(bufr_path_file) + self.logger.info(f'bufr_file_source_path: {bufr_file_source_path}') # Target file ~ conversion output file name (should end in .nc4). # Use the same name but replace the suffix. parts = bufr_file_source_path.rsplit('.', 2) - ioda_file_target_name = parts[0] + '.{splits/satId}.tm00.nc4' + ioda_file_target_name = parts[0] + '.tm00.nc4' ioda_file_target_path = os.path.join(ioda_dir, ioda_file_target_name) + self.logger.info(f'ioda_file_target_path: {ioda_file_target_path}') - bufr2ioda_conv_yaml = self.generate_iodaconv_yaml(bufr_path_file, ioda_file_target_path, + bufr2ioda_conv_yaml = self.get_bufr_mapping_yaml(bufr_path_file, ioda_file_target_path, path_to_ioda_conv_yaml_tmpl_dir) + self.logger.info(f'bufr_path_file: {bufr_path_file}') + self.logger.info(f'bufr2ioda_conv_yaml: {bufr2ioda_conv_yaml}') + self.logger.info(f'ioda_file_target_path: {ioda_file_target_path}') # Jedi executable name (IODA Converter Name) # -------------------- - jedi_executable = 'bufr2ioda.x' + jedi_executable = 'bufr2netcdf.x' jedi_executable_path = os.path.join(self.experiment_path(), 'jedi_bundle', 'build', 'bin', jedi_executable) + + # CLI Command + # ------------ + # bufr2netcdf.x [bufr file] [bufr_mapping.yaml] [output .nc file] + # bufr2netcdf.x bufr_path_file bufr2ioda_conv_yaml ioda_file_target_path + # cli_command = [jedi_executable_path, bufr_path_file, bufr2ioda_conv_yaml, ioda_file_target_path] - # Run the JEDI executable - # ----------------------- try: self.logger.info('Running '+jedi_executable_path+' with '+bufr2ioda_conv_yaml+'.') - subprocess.run([jedi_executable_path, bufr2ioda_conv_yaml]) + subprocess.run([jedi_executable_path, bufr_path_file, bufr2ioda_conv_yaml, ioda_file_target_path]) except FileNotFoundError: self.logger.info(f'Error: File "{bufr2ioda_conv_yaml}" not found.') except yaml.YAMLError as e: self.logger.info(f'Error processing YAML file: {e}') else: - self.logger.info('YAML generated, now exiting.') + self.logger.info('Conversion to ioda complete, now exiting.') + self.logger.info(f'Execution cli line: {jedi_executable_path}, {bufr_path_file}, {bufr2ioda_conv_yaml}, {ioda_file_target_path}') # --------------------------------------------------------------------------------------------------