From 6b4c845669c5b7a7d1b6fee9500dd0bd3fbd66eb Mon Sep 17 00:00:00 2001 From: Shayon Shakoorzadeh Date: Fri, 6 Mar 2026 13:11:30 -0500 Subject: [PATCH 1/8] linkbcs.py beta release --- catalog.yaml | 191 ++++++++++++++++++++++++ linkbcs.py | 400 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 591 insertions(+) create mode 100644 catalog.yaml create mode 100644 linkbcs.py diff --git a/catalog.yaml b/catalog.yaml new file mode 100644 index 00000000..0072135a --- /dev/null +++ b/catalog.yaml @@ -0,0 +1,191 @@ +# inside install/etc/ + +platform: + nccs: + boundary_dir: /discover/nobackup/projects/gmao + atmos_bcs: ./bcs_shared/make_bcs_inputs/atmosphere + chem_dir: fvInput_nc3 + gwdrs_dir: osse2/stage/BCS_FILES/GWD_RIDGE + share_dir: share/gmao_ops + nas: + boundary_dir: /nobackup/gmao_SIteam/ModelData + atmos_bcs: /nas/atmos_bcs/ + chem_dir: fvInput_nc3 + gwdrs_dir: GWD_RIDGE + share_dir: "" + + +land_version: + icarus: + lsm_bcs: ICA + stream: v1 + icarus-NLv3: + lsm_bcs: NL3 + stream: v1 + v12: + lsm_bcs: v12 + stream: v1 + v14: + lsm_bcs: v14 + stream: v2 + + +agcm_grid: + c12: + IM: 12 + JM: 72 + tag: CF0012x6C + c24: + IM: 24 + JM: 144 + tag: CF0024x6C + c48: + IM: 48 + JM: 288 + tag: CF0048x6C + c90: + IM: 90 + JM: 540 + tag: CF0090x6C + c180: + IM: 180 + JM: 1080 + tag: CF0180x6C + c360: + IM: 360 + JM: 2160 + tag: CF0360x6C + c720: + IM: 720 + JM: 4320 + tag: CF0720x6C + c1120: + IM: 1120 + JM: 6720 + tag: CF1120x6C + c1440: + IM: 1440 + JM: 8640 + tag: CF1440x6C + c2880: + IM: 2880 + JM: 17280 + tag: CF2880x6C + c5760: + IM: 5760 + JM: 34560 + tag: CF5760x6C + c270: + IM: 270 + JM: 1620 + tag: CF0270x6C-SG001 + c540: + IM: 540 + JM: 3240 + tag: CF0540x6C-SG001 + c1080: + IM: 1080 + JM: 4320 + tag: CF1080x6C-SG001 + c1536: + IM: 1536 + JM: 9216 + tag: CF1536x6C-SG002 + c2160: + IM: 2160 + JM: 12960 + tag: CF2160x6C-SG001 + c4320: + IM: 4320 + JM: 25920 + tag: CF4320x6C + +ocean_model: + mom6: + o72: + IM: 76 + JM: 36 + o360: + IM: 360 + JM: 210 + o540: + IM: 540 + JM: 458 + o720: + IM: 720 + JM: 576 + o1440: + IM: 1440 + JM: 1080 + o2880: + IM: 2880 + JM: 2240 + ogrid_type: M6TP + data: + reynolds: + IM: 360 + JM: 180 + ogrid_type: DE + sst_name: SST + sst_file: dataoceanfile_MERRA_sst_1971-current + ice_file: dataoceanfile_MERRA_fraci_1971-current + kpar_file: SEAWIFS_KPAR_mon_clim + merra-2: + IM: 1440 + JM: 720 + sst_name: MERRA2 + ogrid_type: DE + sst_file: dataoceanfile_MERRA2_SST + ice_file: dataoceanfile_MERRA2_ICE + kpar_file: SEAWIFS_KPAR_mon_clim + sst_dir: /fvInput/g5gcm/bcs/SST/ + ostia: + IM: 2880 + JM: 1440 + ogrid_type: DE + sst_name: OSTIA_REYNOLDS + sst_file: dataoceanfile_OSTIA_REYNOLDS_SST + ice_file: dataoceanfile_OSTIA_REYNOLDS_ICE + kpar_file: SEAWIFS_KPAR_mon_clim + cubed_sphere_ostia: + ogrid_type: CF + sst_name: OSTIA_REYNOLDS + sst_file: dataoceanfile_OSTIA_REYNOLDS_SST + ice_file: dataoceanfile_OSTIA_REYNOLDS_ICE + kpar_file: SEAWIFS_KPAR_mon_clim + cubed_sphere_ostia_r21c: + ogrid_type: CF + sst_name: OSTIA_REYNOLDS_ITR21C + sst_file: dataoceanfile_OSTIA_REYNOLDS_ITR21C_SST + ice_file: dataoceanfile_OSTIA_REYNOLDS_ITR21C_ICE + kpar_file: SEAWIFS_KPAR_mon_clim + +seaice_model: + cice: + kmt: cice/kmt_cice.bin + grid: cice/grid_cice.bin + cice6: + kmt: cice6/cice6_kmt.nc + grid: cice6/cice6_grid.nc + global_bathy: cice6/cice6_global.bathy.nc + +pchem_species: + ops: + # DAS or REPLAY Mode (AGCM.rc: pchem_clim_years = 1-Year Climatology) + species_data: PCHEM/pchem.species.Clim_Prod_Loss.z_721x72.nc4 + cmip: + # CMIP-5 Ozone Data (AGCM.rc: pchem_clim_years = 228-Years) + species_data: PCHEM/pchem.species.CMIP-5.1870-2097.z_91x72.nc4 + s2s: + # S2S pre-industrial with prod/loss of stratospheric water vapor + # (AGCM.rc: pchem_clim_years = 3-Years, and H2O_ProdLoss: 1 ) + species_data: Shared/pchem.species.CMIP-6.wH2OandPL.1850s.z_91x72.nc4 + merra-2: + # MERRA-2 Ozone Data (AGCM.rc: pchem_clim_years = 39-Years) + species_data: PCHEM/pchem.species.CMIP-5.MERRA2OX.197902-201706.z_91x72.nc4 + +# Is this necessary? +precip_correction: + r21c: /discover/nobackup/projects/gmao/share/gmao_ops/fvInput/merra_land/precip_CPCUexcludeAfrica-CMAP_corrected_MERRA/GEOSdas-2_1_4 + merra-2: /discover/nobackup/projects/gmao/share/gmao_ops/fvInput/merra_land/precip_CPCUexcludeAfrica-CMAP_corrected_MERRA/GEOSdas-2_1_4 + ops: /gpfsm/dnb51/projects/p15/iau/merra_land/precip_CPCU-CMAP_corrected_MERRA/GEOSdas-2_1_4 diff --git a/linkbcs.py b/linkbcs.py new file mode 100644 index 00000000..b33f4f9a --- /dev/null +++ b/linkbcs.py @@ -0,0 +1,400 @@ +import yaml, argparse, sys, shutil +from pathlib import Path +from datetime import datetime + +def validate_iso_datetime(datetime_string): + try: + date = datetime.fromisoformat(datetime_string) + return date + except ValueError: + raise argparse.ArgumentTypeError( + f"Invalid datetime format: '{datetime_string}'. " + f"Expected ISO format (YYYY-MM-DDTHH:MM:SS), e.g., '2025-04-04T00:00:00'" + ) + +def validate_yaml_file(file_path): + path = Path(file_path) + + # Check if file exists + if not path.exists(): + raise argparse.ArgumentTypeError(f"File does not exist: {file_path}") + + # Check file extension + if path.suffix.lower() not in ['.yaml', '.yml']: + raise argparse.ArgumentTypeError(f"File must have .yaml or .yml extension: {file_path}") + + return file_path + +def capture_arguments(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--config", + required=True, + type=validate_yaml_file, + help="User-provided YAML configuration file" + ) + parser.add_argument( + "--timestamp", + required=True, + type=validate_iso_datetime, + help="ISO format date" + ) + + args = parser.parse_args() + return args + + +class CatalogManager: + def __init__(self, config_path: Path): + self.config = self.import_yaml(config_path) + self.catalog = self.import_yaml(Path(self.config['install_dir']) / "etc" / "catalog.yaml") + + def import_yaml(self, yaml_file: Path) -> dict: + with open(yaml_file, 'r') as f: + return yaml.safe_load(f) + + # pulls value from catalog using user config to navigate hierarchy + def get_value(self, target: str, section: str): + catalog_section = self.catalog[section] + return self._search_recursive(target, catalog_section) + + # Helper function for recursively searching through catalog + def _search_recursive(self, target: str, catalog_section: dict): + # Found the target we're looking for + if target in catalog_section: + return catalog_section[target] + + config_values = set(self.config.values()) + + # Search each subdictionary that matches a config value + for section_name, section_data in catalog_section.items(): + + # Skip if not a dictionary + if not isinstance(section_data, dict): + continue + + # Skip if this section name doesn't match any config value + if section_name not in config_values: + continue + + # Recursively search this matching section + result = self._search_recursive(target, section_data) + + # Return immediately if we found something + if result is not None: + return result + + return None + + +class SymlinkCreator: + def __init__(self, catalog: CatalogManager, year: int): + self.catalog = catalog + self.config = catalog.config + self.year = year + + # platform + self.boundary_dir = Path(catalog.get_value('boundary_dir', 'platform')) + self.atmos_bcs = catalog.get_value('atmos_bcs', 'platform') + self.gwdrs_dir = self.boundary_dir / catalog.get_value('gwdrs_dir', 'platform') + self.share_dir = catalog.get_value('share_dir', 'platform') + self.chem_dir = self.boundary_dir / self.share_dir / catalog.get_value('chem_dir', 'platform') + + # land + self.stream = catalog.get_value('stream', 'land_version') + self.bcs_dir = self.boundary_dir / "bcs_shared/fvInput/ExtData/esm/tiles" / catalog.get_value('lsm_bcs', 'land_version') + + # atmos + self.atmos_tag = catalog.get_value('tag', 'agcm_grid') + self.agcm_IM = catalog.get_value('IM', 'agcm_grid') + self.agcm_JM = catalog.get_value('JM', 'agcm_grid') + + # ocean + if self.config["ocean_model"] != "data": + self.coupled = True + else: + self.coupled = False + self.ogrid_type = catalog.get_value('ogrid_type', 'ocean_model') + self.ogcm_IM = catalog.get_value('IM', 'ocean_model') + self.ogcm_JM = catalog.get_value('JM', 'ocean_model') + if self.config['ogcm_grid'] == 'cubed_sphere_ostia': + self.ogcm_IM = self.agcm_IM + self.ogcm_JM = self.agcm_JM + self.ocean_res = f"{self.ogcm_IM}x{self.ogcm_JM}" + self.coupled_dir = self.boundary_dir / f"bcs_shared/make_bcs_inputs/ocean/{self.config['ocean_model'].upper()}" + self.tables = catalog.get_value('tables', 'ocean_model') + self.sst_name = catalog.get_value('sst_name', 'ocean_model') + self.sst_file = catalog.get_value('sst_file', 'ocean_model') + self.ice_file = catalog.get_value('ice_file', 'ocean_model') + self.kpar_file = catalog.get_value('kpar_file', 'ocean_model') + + # seaice + self.kmt_cice = catalog.get_value('kmt', 'seaice_model') + self.grid_cice = catalog.get_value('grid', 'seaice_model') + self.global_bathy = catalog.get_value('global_bathy', 'seaice_model') + + # pchem_species + self.species_data_dir = self.bcs_dir / catalog.get_value('species_data', 'pchem_species') + + # precip correction + self.precip_dir = Path(self.catalog.get_value('merra-2', 'precip_correction')) + + # misc + self.bcrslv = f"{self.atmos_tag}_{self.ogrid_type}{str(self.ogcm_IM).zfill(4)}x{str(self.ogcm_JM).zfill(4)}" + if self.config['ogcm_grid'] == "cubed_sphere_ostia": + self.bcrslv = f"{self.atmos_tag}_CF{str(self.ogcm_IM).zfill(4)}x6C" + elif self.config['ocean_model'] == "data": + self.bcrslv = f"{self.atmos_tag}_DE{str(self.ogcm_IM).zfill(4)}xPE{str(self.ogcm_JM).zfill(4)}" + self.topo_src_dir = self.boundary_dir / self.atmos_bcs / "TOPO" / self.stream / self.atmos_tag / "smoothed" + + # sst_dir + if not self.coupled and self.ocean_res == "1440x720": + self.sst_dir = self.boundary_dir / self.share_dir / f"fvInput/g5gcm/bcs/SST/{self.ocean_res}" + elif not self.coupled: + self.sst_dir = self.boundary_dir / self.share_dir / f"fvInput/g5gcm/bcs/realtime/{self.sst_name}/{self.ocean_res}" + if self.config['platform'] != "nas" and self.config['platform'] != "nccs" and not self.coupled: + self.sst_dir = self.boundary_dir / self.sst_name / self.ocean_res + elif self.coupled: + self.sst_dir = self.coupled_dir / f"SST/MERRA2/{self.ocean_res}/v1" + else: + #exception + pass + + + def create_symlink(self, symlink_name: Path, file_path: Path): + # remove existing link if it exists (equivalent of -f flag) + if symlink_name.is_symlink(): + symlink_name.unlink() + + symlink_name.symlink_to(file_path) + + def topo_paths(self) -> dict: + paths = { + "topo_dynave.data": self.topo_src_dir / f"topo_DYN_ave_{self.agcm_IM}x{self.agcm_JM}.data", + "topo_gwdvar.data": self.topo_src_dir / f"topo_GWD_var_{self.agcm_IM}x{self.agcm_JM}.data", + "topo_trbvar.data": self.topo_src_dir / f"topo_TRB_var_{self.agcm_IM}x{self.agcm_JM}.data" + } + + return paths + + def land_paths(self) -> dict: + land_src_dir = self.bcs_dir / "land" / self.bcrslv + paths = { + "visdf.dat": land_src_dir / f"visdf_{self.agcm_IM}x{self.agcm_JM}.dat", + "nirdf.dat": land_src_dir / f"nirdf_{self.agcm_IM}x{self.agcm_JM}.dat", + "vegdyn.data": land_src_dir / f"vegdyn_{self.agcm_IM}x{self.agcm_JM}.dat", # .dat or .data? + "lai_clim.data": land_src_dir / f"lai_clim_{self.agcm_IM}x{self.agcm_JM}.data", + "green_clim.data": land_src_dir / f"green_clim_{self.agcm_IM}x{self.agcm_JM}.data", + "ndvi_clim.data": land_src_dir / f"ndvi_clim_{self.agcm_IM}x{self.agcm_JM}.data" + } + + return paths + + def make_restart_dir(self): + if self.coupled != "data": + Path("RESTART").mkdir(parents=True, exist_ok=True) + + def make_extdata_dir(self): + extdata = Path("ExtData") + extdata.mkdir(parents=True, exist_ok=True) + + for item in self.chem_dir.glob("*"): + self.create_symlink(extdata / item.name, self.chem_dir / item.name) + + # exit here if not coupled ocean + if not self.coupled: + return + dataatm_dir = self.boundary_dir / f"bcs_shared/make_bcs_inputs/ocean/dataatm" + for item in dataatm_dir.glob("*"): + self.create_symlink(extdata / item.name, dataatm_dir / item.name) + + + def seawifs_path(self) -> dict: + if not self.coupled: + return {} + paths = {"SEAWIFS_KPAR_mon_clim.data": self.coupled_dir / f"{self.ogcm_IM}x{self.ogcm_JM}/SEAWIFS_KPAR_mon_clim.{self.ogcm_IM}x{self.ogcm_JM}"} + return paths + + def tile_paths(self) -> dict: + tile_data = self.bcs_dir / "geometry" / self.bcrslv / f"{self.bcrslv}-Pfafstetter.til" + tile_bin = self.bcs_dir / "geometry" / self.bcrslv / f"{self.bcrslv}-Pfafstetter.TIL" + paths = {"tile.data": tile_data} + if not self.coupled and tile_bin.exists(): + paths["tile.bin"] = tile_bin + + return paths + + def runoff_path(self) -> dict: + if not self.coupled: + return {} + paths = {"runoff.bin": self.bcs_dir / "geometry" / self.bcrslv / f"{self.bcrslv}-Pfafstetter.TRN"} + return paths + + def mapl_tripolar_path(self) -> dict: + if self.config["ocean_model"] != "mom": + return {} + paths = {"MAPL_Tripolar.nc": self.coupled_dir / f"{self.ogcm_IM}x{self.ogcm_JM}/MAPL_Tripolar.nc"} + return paths + + def vgrid_path(self) -> dict: + if self.config["ocean_model"] != "mom": + return {} + + if self.agcm_IM == 12 or self.agcm_IM == 90: + ogcm_LM = 50 + elif self.agcm_IM == 180: + ogcm_LM = 75 + else: + sys.exit("ERROR: must use c12, c90, or c180 with MOM6!") + + paths = {"vgrid.ascii": self.coupled_dir / f"{self.ogcm_IM}x{self.ogcm_JM}/vgrid{ogcm_LM}_LM.ascii"} + return paths + + def MIT_paths(self) -> dict: + if self.config["ocean_model"] != "MIT": + return {} + + paths = { + "mit.ascii": self.bcs_dir / f"geometry/{self.bcrslv}/mit.ascii", + "DC0360xPC0181_LL5400x15-LL.bin": self.coupled_dir / "DC0360xPC0181_LL5400x15-LL.bin" + } + + return paths + + def precip_path(self) -> dict: + if not self.config["precip_correction"]: + return {} + paths = {"ExtData/PCP": self.precip_dir} + return paths + + def species_path(self) -> dict: + path = {"species.data": self.species_data_dir} + return path + + def catchcn_paths(self) -> dict: + if not self.config["catchcn"]: + return {} + + paths = {} + lnfm_data = self.bcs_dir / self.bcrslv / f"lnfm_clim_{self.agcm_IM}x{self.agcm_JM}.data" + if lnfm_data.exists(): + paths["lnfm.data"] = lnfm_data + + paths["CO2_MonthlyMean_DiurnalCycle.nc4"] = self.bcs_dir / "land/shared/CO2_MonthlyMean_DiurnalCycle.nc4" + return paths + + # Optional internal restart + # COPY (not symlinking these) + def copy_internal_restart(self): + gwd_rst = self.topo_src_dir / "gwd_internal_rst" + gwd_agcm = self.gwdrs_dir / f"gwd_internal_c{self.agcm_IM}" + path = {} + if gwd_rst.exists(): + shutil.copy(gwd_rst, Path.cwd()) + elif gwd_agcm.exists(): + shutil.copy(gwd_agcm, Path.cwd()) + + + def table_paths(self) -> dict: + if self.config["ocean_model"] != "mom": + return {} + paths = { + "diag_table": Path(self.config["install_dir"]) / f"etc/MOM6/mom6_app/{self.ogcm_IM}x{self.ogcm_JM}/diag_table", + "data_table": Path(self.config["install_dir"]) / f"etc/MOM6/mom6_app/{self.ogcm_IM}x{self.ogcm_JM}/data_table" + } + return paths + + def make_input_dir(self): + if not self.coupled: + return {} + + src_dir = self.coupled_dir / f"{self.ogcm_IM}x{self.ogcm_JM}/INPUT" + target_dir = Path("INPUT") + + # make input dir if it doesn't already exist + Path("INPUT").mkdir(parents=True, exist_ok=True) + + for file_path in src_dir.glob("*"): + if file_path.is_file(): + # copy2 preserves file metadata + shutil.copy2(file_path, target_dir / file_path.name) + + def seaice_paths(self) -> dict: + if not self.coupled: + return {} + if self.config["seaice_model"] == "cice4": + paths = { + "kmt_cice.bin": self.coupled_dir / f"{self.ogcm_IM}x{self.ogcm_JM}" / self.kmt_cice, + "grid_cice.bin": self.coupled_dir / f"{self.ogcm_IM}x{self.ogcm_JM}" / self.grid_cice + } + elif self.config["seaice_model"] == "cice6": + paths = { + "cice6_grid.nc": self.coupled_dir / f"{self.ogcm_IM}x{self.ogcm_JM}" / self.kmt_cice, + "cice6_kmt.nc": self.coupled_dir / f"{self.ogcm_IM}x{self.ogcm_JM}" / self.grid_cice, + "cice6_global.bathy.nc": self.coupled_dir / f"{self.ogcm_IM}x{self.ogcm_JM}" / self.global_bathy + } + return paths + + def dataocean_paths(self) -> dict: + if self.coupled: + return {} + + if self.config["ogcm_grid"] == "reynolds": + paths = { + "sst.data": self.sst_dir / f"{self.sst_file}.{self.ocean_res}.LE", + "fraci.data": self.sst_dir / f"{self.ice_file}.{self.ocean_res}.LE" + } + else: + paths = { + "sst.data": self.sst_dir / f"{self.sst_file}.{self.ocean_res}.{self.year}.data", + "fraci.data": self.sst_dir / f"{self.ice_file}.{self.ocean_res}.{self.year}.data" + } + paths["SEAWIFS_KPAR_mon_clim.data"] = self.sst_dir / f"{self.kpar_file}.{self.ocean_res}" + + return paths + + def make_symlinks(self): + paths = {} + paths.update(self.topo_paths()) + paths.update(self.land_paths()) + paths.update(self.seawifs_path()) + paths.update(self.tile_paths()) + paths.update(self.runoff_path()) + paths.update(self.mapl_tripolar_path()) + paths.update(self.vgrid_path()) + paths.update(self.MIT_paths()) + paths.update(self.precip_path()) + paths.update(self.species_path()) + paths.update(self.catchcn_paths()) + paths.update(self.table_paths()) + paths.update(self.seaice_paths()) + paths.update(self.dataocean_paths()) + #self.print_paths(paths) + + for items in paths: + self.create_symlink(Path(items), paths[items]) + + self.make_restart_dir() + self.make_extdata_dir() + self.copy_internal_restart() + self.make_input_dir() + + # helper for testing + def print_paths(self, paths): + for i in paths: + if paths[i].exists(): + print(f"{i}: {paths[i]}") + else: + print(f"!!!!!!!{i} does not exist!\n{paths[i]}") + + + +def main(): + args = capture_arguments() + catalog_manager = CatalogManager(Path(args.config)) + symlink_creator = SymlinkCreator(catalog_manager, args.timestamp.year) + + symlink_creator.make_symlinks() + +if __name__ == "__main__": + main() From f0daa9d24d22375afba89e22d860c9e542bfeb57 Mon Sep 17 00:00:00 2001 From: Shayon Shakoorzadeh Date: Fri, 6 Mar 2026 13:54:33 -0500 Subject: [PATCH 2/8] modded cmakelists and fixed land symlink names --- CMakeLists.txt | 6 ++++++ linkbcs.py | 8 ++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 76895e3a..3b6e78e1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,6 +39,7 @@ set (programs gcm_quickstat.j scm_run.j construct_extdata_yaml_list.py + linkbcs.py ) install ( @@ -46,6 +47,11 @@ install ( DESTINATION etc ) +install ( + FILES catalog.yaml + DESTINATION etc + ) + install ( PROGRAMS ${programs} DESTINATION bin diff --git a/linkbcs.py b/linkbcs.py index b33f4f9a..75256f61 100644 --- a/linkbcs.py +++ b/linkbcs.py @@ -182,10 +182,10 @@ def land_paths(self) -> dict: paths = { "visdf.dat": land_src_dir / f"visdf_{self.agcm_IM}x{self.agcm_JM}.dat", "nirdf.dat": land_src_dir / f"nirdf_{self.agcm_IM}x{self.agcm_JM}.dat", - "vegdyn.data": land_src_dir / f"vegdyn_{self.agcm_IM}x{self.agcm_JM}.dat", # .dat or .data? - "lai_clim.data": land_src_dir / f"lai_clim_{self.agcm_IM}x{self.agcm_JM}.data", - "green_clim.data": land_src_dir / f"green_clim_{self.agcm_IM}x{self.agcm_JM}.data", - "ndvi_clim.data": land_src_dir / f"ndvi_clim_{self.agcm_IM}x{self.agcm_JM}.data" + "vegdyn.data": land_src_dir / f"vegdyn_{self.agcm_IM}x{self.agcm_JM}.dat", + "lai.data": land_src_dir / f"lai_clim_{self.agcm_IM}x{self.agcm_JM}.data", + "green.data": land_src_dir / f"green_clim_{self.agcm_IM}x{self.agcm_JM}.data", + "ndvi.data": land_src_dir / f"ndvi_clim_{self.agcm_IM}x{self.agcm_JM}.data" } return paths From e8fc61a6aadc0a1a734685b3e12375ab448e9f01 Mon Sep 17 00:00:00 2001 From: Shayon Shakoorzadeh Date: Thu, 12 Mar 2026 20:05:26 -0400 Subject: [PATCH 3/8] added ADAS support --- catalog.yaml | 39 +++++++++++++++++++++++++++++++-------- linkbcs.py | 32 +++++++++++++++++++------------- 2 files changed, 50 insertions(+), 21 deletions(-) diff --git a/catalog.yaml b/catalog.yaml index 0072135a..941dd83a 100644 --- a/catalog.yaml +++ b/catalog.yaml @@ -2,17 +2,37 @@ platform: nccs: - boundary_dir: /discover/nobackup/projects/gmao - atmos_bcs: ./bcs_shared/make_bcs_inputs/atmosphere - chem_dir: fvInput_nc3 - gwdrs_dir: osse2/stage/BCS_FILES/GWD_RIDGE - share_dir: share/gmao_ops + boundary_dir: /discover/nobackup/projects/gmao/share/gmao_ops + gwdrs_dir: GWD_RIDGE nas: boundary_dir: /nobackup/gmao_SIteam/ModelData - atmos_bcs: /nas/atmos_bcs/ - chem_dir: fvInput_nc3 gwdrs_dir: GWD_RIDGE - share_dir: "" + atmos_bcs: ./bcs_shared/make_bcs_inputs/atmosphere + + +experiment_type: + GCM: + extdata_files: + - AeroCom + - MERRA2 + - PIESA + - chemistry + - g5chem + - g5gcm + chem_dir: fvInput_nc3 + fvInput_dir: fvInput + ADAS: + extdata_files: + - AeroCom + - MERRA2 + - PIESA + - agcmpert + - chemistry + - g5chem + - g5gcm + chem_dir: fvInput_4dvar + fvInput_dir: fvInput_4dvar + land_version: @@ -189,3 +209,6 @@ precip_correction: r21c: /discover/nobackup/projects/gmao/share/gmao_ops/fvInput/merra_land/precip_CPCUexcludeAfrica-CMAP_corrected_MERRA/GEOSdas-2_1_4 merra-2: /discover/nobackup/projects/gmao/share/gmao_ops/fvInput/merra_land/precip_CPCUexcludeAfrica-CMAP_corrected_MERRA/GEOSdas-2_1_4 ops: /gpfsm/dnb51/projects/p15/iau/merra_land/precip_CPCU-CMAP_corrected_MERRA/GEOSdas-2_1_4 + + + diff --git a/linkbcs.py b/linkbcs.py index 75256f61..34571632 100644 --- a/linkbcs.py +++ b/linkbcs.py @@ -97,9 +97,12 @@ def __init__(self, catalog: CatalogManager, year: int): self.boundary_dir = Path(catalog.get_value('boundary_dir', 'platform')) self.atmos_bcs = catalog.get_value('atmos_bcs', 'platform') self.gwdrs_dir = self.boundary_dir / catalog.get_value('gwdrs_dir', 'platform') - self.share_dir = catalog.get_value('share_dir', 'platform') - self.chem_dir = self.boundary_dir / self.share_dir / catalog.get_value('chem_dir', 'platform') - + + # experiment type + self.extdata_files = catalog.get_value('extdata_files', 'experiment_type') + self.chem_dir = self.boundary_dir / catalog.get_value('chem_dir', 'experiment_type') + self.fvInput_dir = catalog.get_value('fvInput_dir', 'experiment_type') + # land self.stream = catalog.get_value('stream', 'land_version') self.bcs_dir = self.boundary_dir / "bcs_shared/fvInput/ExtData/esm/tiles" / catalog.get_value('lsm_bcs', 'land_version') @@ -149,9 +152,9 @@ def __init__(self, catalog: CatalogManager, year: int): # sst_dir if not self.coupled and self.ocean_res == "1440x720": - self.sst_dir = self.boundary_dir / self.share_dir / f"fvInput/g5gcm/bcs/SST/{self.ocean_res}" + self.sst_dir = self.boundary_dir / self.fvInput_dir / f"g5gcm/bcs/SST/{self.ocean_res}" elif not self.coupled: - self.sst_dir = self.boundary_dir / self.share_dir / f"fvInput/g5gcm/bcs/realtime/{self.sst_name}/{self.ocean_res}" + self.sst_dir = self.boundary_dir / self.fvInput_dir / f"g5gcm/bcs/realtime/{self.sst_name}/{self.ocean_res}" if self.config['platform'] != "nas" and self.config['platform'] != "nccs" and not self.coupled: self.sst_dir = self.boundary_dir / self.sst_name / self.ocean_res elif self.coupled: @@ -198,8 +201,8 @@ def make_extdata_dir(self): extdata = Path("ExtData") extdata.mkdir(parents=True, exist_ok=True) - for item in self.chem_dir.glob("*"): - self.create_symlink(extdata / item.name, self.chem_dir / item.name) + for file in self.extdata_files: + self.create_symlink(extdata / file, self.chem_dir / file) # exit here if not coupled ocean if not self.coupled: @@ -269,6 +272,7 @@ def precip_path(self) -> dict: def species_path(self) -> dict: path = {"species.data": self.species_data_dir} + return path def catchcn_paths(self) -> dict: @@ -369,7 +373,8 @@ def make_symlinks(self): paths.update(self.table_paths()) paths.update(self.seaice_paths()) paths.update(self.dataocean_paths()) - #self.print_paths(paths) + + self.print_paths(paths) for items in paths: self.create_symlink(Path(items), paths[items]) @@ -379,14 +384,15 @@ def make_symlinks(self): self.copy_internal_restart() self.make_input_dir() - # helper for testing + # returns broken paths and exits (if they exist) def print_paths(self, paths): + broken_path = False for i in paths: - if paths[i].exists(): - print(f"{i}: {paths[i]}") - else: + if not paths[i].exists(): print(f"!!!!!!!{i} does not exist!\n{paths[i]}") - + broken_path = True + if broken_path: + sys.exit() def main(): From 878a7d186ec1066d30202277d919679f60029867 Mon Sep 17 00:00:00 2001 From: Shayon Shakoorzadeh Date: Fri, 13 Mar 2026 10:56:11 -0400 Subject: [PATCH 4/8] added shebang run option --- linkbcs.py | 2 ++ 1 file changed, 2 insertions(+) mode change 100644 => 100755 linkbcs.py diff --git a/linkbcs.py b/linkbcs.py old mode 100644 new mode 100755 index 34571632..4e1b9fd3 --- a/linkbcs.py +++ b/linkbcs.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + import yaml, argparse, sys, shutil from pathlib import Path from datetime import datetime From d93656edeb9550b3a44d35855689af832be1ac67 Mon Sep 17 00:00:00 2001 From: Shayon Shakoorzadeh Date: Mon, 16 Mar 2026 14:06:25 -0400 Subject: [PATCH 5/8] created a helper script for packtime to iso format --- packtoiso.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100755 packtoiso.py diff --git a/packtoiso.py b/packtoiso.py new file mode 100755 index 00000000..7c0644d2 --- /dev/null +++ b/packtoiso.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 + +import sys, datetime + +def main(): + with open(sys.argv[1], 'r') as file: + content = file.read().strip() + + date, time = content.split() + year = int(date[:4]) + month = int(date[4:6]) + day = int(date[6:8]) + hour = int(time[:2]) + minute = int(time[2:4]) + second = int(time[4:6]) + + iso_dt = datetime.datetime(year, month, day, hour, minute, second) + print(iso_dt.strftime('%Y-%m-%dT%H:%M:%S')) + +if __name__ == "__main__": + main() From 30c20f0c9c9825fa526098dd605036dc4a6c5b30 Mon Sep 17 00:00:00 2001 From: Shayon Shakoorzadeh Date: Mon, 16 Mar 2026 14:36:10 -0400 Subject: [PATCH 6/8] added gmao_desktop to catalog --- catalog.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/catalog.yaml b/catalog.yaml index 941dd83a..db20c3f2 100644 --- a/catalog.yaml +++ b/catalog.yaml @@ -1,12 +1,11 @@ -# inside install/etc/ - platform: nccs: boundary_dir: /discover/nobackup/projects/gmao/share/gmao_ops - gwdrs_dir: GWD_RIDGE nas: boundary_dir: /nobackup/gmao_SIteam/ModelData - gwdrs_dir: GWD_RIDGE + gmao_desktop: + boundary_dir: /ford1/share/gmao_SIteam/ModelData + gwdrs_dir: GWD_RIDGE atmos_bcs: ./bcs_shared/make_bcs_inputs/atmosphere From c7c4e12a831c15a8f75cb47a84d26f883daa700e Mon Sep 17 00:00:00 2001 From: Shayon Shakoorzadeh Date: Mon, 16 Mar 2026 15:32:14 -0400 Subject: [PATCH 7/8] updated cmakelists.txt --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3b6e78e1..705159b2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,6 +40,7 @@ set (programs scm_run.j construct_extdata_yaml_list.py linkbcs.py + packtoiso.py ) install ( From 4156f58ec565b5c658fa76344b26195105d53cea Mon Sep 17 00:00:00 2001 From: Shayon Shakoorzadeh Date: Tue, 17 Mar 2026 15:30:52 -0400 Subject: [PATCH 8/8] renamed catalog to bcs_catalog --- CMakeLists.txt | 2 +- catalog.yaml => bcs_catalog.yaml | 2 ++ linkbcs.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) rename catalog.yaml => bcs_catalog.yaml (99%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 705159b2..7f8b1545 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -49,7 +49,7 @@ install ( ) install ( - FILES catalog.yaml + FILES bcs_catalog.yaml DESTINATION etc ) diff --git a/catalog.yaml b/bcs_catalog.yaml similarity index 99% rename from catalog.yaml rename to bcs_catalog.yaml index db20c3f2..4e6ca9ad 100644 --- a/catalog.yaml +++ b/bcs_catalog.yaml @@ -1,3 +1,5 @@ +# inside install/etc/ + platform: nccs: boundary_dir: /discover/nobackup/projects/gmao/share/gmao_ops diff --git a/linkbcs.py b/linkbcs.py index 4e1b9fd3..cc8aba14 100755 --- a/linkbcs.py +++ b/linkbcs.py @@ -49,7 +49,7 @@ def capture_arguments(): class CatalogManager: def __init__(self, config_path: Path): self.config = self.import_yaml(config_path) - self.catalog = self.import_yaml(Path(self.config['install_dir']) / "etc" / "catalog.yaml") + self.catalog = self.import_yaml(Path(self.config['install_dir']) / "etc" / "bcs_catalog.yaml") def import_yaml(self, yaml_file: Path) -> dict: with open(yaml_file, 'r') as f: