diff --git a/CHANGELOG.md b/CHANGELOG.md index 24cb320f..902e9c24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,11 +16,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added new res C1120 - NOTE: If running on SLES15 remap tests will not be zero diff for GOCART RST but are zero diff for all other - Add support for v14 BCs +- Add SLURM/PBS job submission capability to lake-landice remapping, bringing it in line with upper-air and catch remapping (batch queues on both NAS and NCCS) +- Add `reservation` field to `remap_params.tpl` and all test YAML files +- Add `split_saltwater` field to `remap_params.tpl` and all test YAML files ### Changed - Update `movestat` to allow for multiple seasons - Update `remap_restarts.py` to first ask about remapping from GEOS-IT, then ask about MERRA-2. Add detection of MERRA-2 path as well +- Restrict SLURM `--constraint` to `mil` (milan) only at NCCS in `remap_catchANDcn.py`, `remap_lake_landice_saltwater.py`, and `remap_upper.py` ### Fixed @@ -28,6 +32,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Due to GRADS variable name character limits, the actual or climatology labels for some plots were being inaccurately overwritten if models names began with the same 8 characters; these variable names now utilize expid numbers instead of model names - Corrected LWP plot command in portrait.script to only include LWP not CCWP - Edited gcmpost.script to move up the location of $SOURCE/plot/.quickplotrc to restore missing plots from landscape.list for some users. +- Fix `{Bin}` unexpanded string in `remap_lake_landice_saltwater.py` by substituting the resolved `bindir` variable before constructing `esma_mpirun` command strings +- Remove forced `-f` flag from `nccmp` comparison in `test_remap_restarts.py` ### Removed diff --git a/pre/remap_restart/remap_catchANDcn.py b/pre/remap_restart/remap_catchANDcn.py index d4a7ed22..41aaaa6e 100755 --- a/pre/remap_restart/remap_catchANDcn.py +++ b/pre/remap_restart/remap_catchANDcn.py @@ -154,7 +154,7 @@ def remap(self): if (qos != ''): QOS = "#SBATCH --qos="+qos - CONSTRAINT = '"[cas|mil]"' + CONSTRAINT = 'mil' account = config['slurm_pbs']['account'] diff --git a/pre/remap_restart/remap_lake_landice_saltwater.py b/pre/remap_restart/remap_lake_landice_saltwater.py index 99df15ca..474f5485 100755 --- a/pre/remap_restart/remap_lake_landice_saltwater.py +++ b/pre/remap_restart/remap_lake_landice_saltwater.py @@ -16,284 +16,561 @@ import glob import ruamel.yaml import shlex -from remap_base import remap_base -from remap_utils import get_label, get_geomdir, get_zoom, MERRA2_RST_BASE +from remap_base import remap_base +from remap_utils import ( + get_label, + get_geomdir, + get_zoom, + get_landdir, + job_directive, + GEOS_SITE, + MERRA2_RST_BASE, +) from remap_bin2nc import bin2nc + class lake_landice_saltwater(remap_base): - def __init__(self, **configs): - super().__init__(**configs) - if self.config['input']['shared']['MERRA-2']: - self.copy_merra2() - if self.config['input']['shared']['GEOS-IT']: - self.copy_geosit() - - def remap(self): - if not self.config['output']['surface']['remap_water']: - return - - restarts_in = self.find_rst() - if len(restarts_in) == 0: - return - - print("\nRemapping land, landice, saltwater.....\n") - config = self.config - cwdir = os.getcwd() - bindir = os.path.dirname(os.path.realpath(__file__)) - - in_bc_base = config['input']['shared']['bc_base'] - in_bc_version = config['input']['shared']['bc_version'] - out_bc_base = config['output']['shared']['bc_base'] - out_bc_version= config['output']['shared']['bc_version'] - - agrid = config['input']['shared']['agrid'] - ogrid = config['input']['shared']['ogrid'] - omodel = config['input']['shared']['omodel'] - stretch = config['input']['shared']['stretch'] - in_tile_file = config['input']['surface']['catch_tilefile'] - if not in_tile_file : - EASE_grid = config['input']['surface'].get('EASE_grid', None) - in_geomdir = get_geomdir(in_bc_base, in_bc_version, agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch, grid=EASE_grid) - in_tile_file = glob.glob(in_geomdir+ '/*.til')[0] - - agrid = config['output']['shared']['agrid'] - ogrid = config['output']['shared']['ogrid'] - omodel = config['output']['shared']['omodel'] - stretch = config['output']['shared']['stretch'] - out_tile_file = config['output']['surface']['catch_tilefile'] - if not out_tile_file : - EASE_grid = config['output']['surface'].get('EASE_grid', None) - out_geomdir = get_geomdir(out_bc_base, out_bc_version, agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch, grid=EASE_grid) - out_tile_file = glob.glob(out_geomdir+ '/*.til')[0] - - types = '.bin' - type_str = sp.check_output(['file','-b', os.path.realpath(restarts_in[0])]) - type_str = str(type_str) - if 'Hierarchical' in type_str: - types = '.nc4' - yyyymmddhh_ = str(config['input']['shared']['yyyymmddhh']) - - label = get_label(config) - suffix = yyyymmddhh_[0:8]+'_'+yyyymmddhh_[8:10] +'z' + types + label - - out_dir = config['output']['shared']['out_dir'] - expid = config['output']['shared']['expid'] - if (expid) : - expid = expid + '.' - else: - expid = '' - - no_remap = self.copy_without_remap(restarts_in, in_tile_file, out_tile_file, suffix) - if no_remap : return - - if not os.path.exists(out_dir) : os.makedirs(out_dir) - print( "cd " + out_dir) - os.chdir(out_dir) - - InData_dir = out_dir+'/InData/' - if os.path.exists(InData_dir) : sp.call(['rm', '-rf',InData_dir]) - print ("mkdir " + InData_dir) - os.makedirs(InData_dir) - - OutData_dir = out_dir+'/OutData/' - if os.path.exists(OutData_dir) : sp.call(['rm', '-rf',OutData_dir]) - print ("mkdir " + OutData_dir) - os.makedirs(OutData_dir) - - - saltwater_internal = '' - saltwater_import = '' - seaicethermo_internal = '' - seaicethermo_import = '' - landice = '' - lake = '' - route = '' - openwater = '' - for rst in restarts_in: - f = os.path.basename(rst) - dest = InData_dir+'/'+f - if os.path.exists(dest) : shutil.remove(dest) - print('\nCopy ' + rst + ' to ' +dest) - shutil.copy(rst,dest) - if 'saltwater_internal' in f : saltwater_internal = f - if 'saltwater_import' in f : saltwater_import = f - if 'seaicethermo_internal' in f : seaicethermo_internal = f - if 'seaicethermo_import' in f : seaicethermo_import = f - if 'landice' in f : landice = f - if 'lake' in f : lake = f - if 'roue' in f : route = f - if 'openwater' in f : openwater = f - - in_til = InData_dir+'/' + os.path.basename(in_tile_file) - out_til = OutData_dir+'/'+ os.path.basename(out_tile_file) - - if os.path.exists(in_til) : shutil.remove(in_til) - if os.path.exists(out_til) : shutil.remove(out_til) - print('\n Copy ' + in_tile_file + ' to ' + in_til) - shutil.copy(in_tile_file, in_til) - print('\n Copy ' + out_tile_file + ' to ' + out_til) - shutil.copy(out_tile_file, out_til) - - exe = bindir + '/mk_LakeLandiceSaltRestarts.x ' - zoom = config['input']['surface']['zoom'] - if zoom is None : - zoom = get_zoom(config) - - log_name = out_dir+'/remap_lake_landice_saltwater_log' - if os.path.exists(log_name): - os.remove(log_name) - - if (saltwater_internal): - cmd = exe + out_til + ' ' + in_til + ' InData/'+ saltwater_internal + ' 0 ' + str(zoom) - self.run_and_log(cmd, log_name) - - # split Saltwater Internal - # NOTE: split_saltwater==True means that the input restarts are already split. - # So we do not split them again. - if not config['input']['surface']['split_saltwater']: - print("\nSplitting Saltwater Internal...\n") - cmd = bindir+'/SaltIntSplitter.x ' + out_til + ' ' + 'OutData/' + saltwater_internal -# subprocess.call(shlex.split(cmd)) - openwater = '' - seaice = '' - self.run_and_log(cmd, log_name) - - # We can now *remove* the unsplit saltwater internal restart to avoid confusion - unsplit_file = 'OutData/' + saltwater_internal - if os.path.exists(unsplit_file): - print('\n Removing unsplit saltwater internal restart: ' + unsplit_file + '\n') - os.remove(unsplit_file) - - - if (saltwater_import): - cmd = exe + out_til + ' ' + in_til + ' InData/'+ saltwater_import + ' 0 ' + str(zoom) - self.run_and_log(cmd, log_name) - - # split Saltwater Import - # NOTE: split_saltwater==True means that the input restarts are already split. - # So we do not split them again. - if not config['input']['surface']['split_saltwater']: - print("\nSplitting Saltwater Import...\n") - cmd = bindir+'/SaltIntSplitter.x ' + out_til + ' ' + 'OutData/' + saltwater_import -# subprocess.call(shlex.split(cmd)) - openwater = '' - seaice = '' - self.run_and_log(cmd, log_name) - - - if (openwater): - cmd = exe + out_til + ' ' + in_til + ' InData/' + openwater + ' 0 ' + str(zoom) - self.run_and_log(cmd, log_name) - - if (seaicethermo_internal): - cmd = exe + out_til + ' ' + in_til + ' InData/' + seaicethermo_internal + ' 0 ' + str(zoom) - print('\n'+cmd) - self.run_and_log(cmd, log_name) - - if (seaicethermo_import): - cmd = exe + out_til + ' ' + in_til + ' InData/' + seaicethermo_import + ' 0 ' + str(zoom) - print('\n'+cmd) - self.run_and_log(cmd, log_name) - - if (lake): - cmd = exe + out_til + ' ' + in_til + ' InData/' + lake + ' 19 ' + str(zoom) - self.run_and_log(cmd, log_name) - - if (landice): - cmd = exe + out_til + ' ' + in_til + ' InData/' + landice + ' 20 ' + str(zoom) - self.run_and_log(cmd, log_name) - - if (route): - route = bindir + '/mk_RouteRestarts.x ' - cmd = route + out_til + ' ' + yyyymmddhh_[0:6] - self.run_and_log(cmd, log_name) - - suffix = '_rst.' + suffix - for out_rst in glob.glob("OutData/*_rst*"): - filename = expid + os.path.basename(out_rst).split('_rst')[0].split('.')[-1]+suffix - print('\n Move ' + out_rst + ' to ' + out_dir+"/"+filename) - shutil.move(out_rst, out_dir+"/"+filename) - print('cd ' + cwdir) - os.chdir(cwdir) - - if self.config['input']['shared']['MERRA-2']: - self.remove_merra2() - - def run_and_log(self, cmd, log_name): - print('\n'+cmd) - process =sp.Popen(shlex.split(cmd), stdout=sp.PIPE, stderr=sp.PIPE) - stdout, stderr = process.communicate() - stdout = stdout.decode() - stderr = stderr.decode() - print('\n'+ stdout) - print('\n'+ stderr) - with open(log_name, "a") as log_: - log_.write(cmd) - log_.write(stdout) - log_.write(stderr) - - def find_rst(self): - surf_restarts =[ - "route_internal_rst" , - "lake_internal_rst" , - "landice_internal_rst" , - "openwater_internal_rst" , - "saltwater_internal_rst" , - "saltwater_import_rst" , - "seaicethermo_internal_rst", - "seaicethermo_import_rst"] - - rst_dir = self.config['input']['shared']['rst_dir'] - yyyymmddhh_ = str(self.config['input']['shared']['yyyymmddhh']) - time = yyyymmddhh_[0:8]+'_'+yyyymmddhh_[8:10] - restarts_in=[] - for f in surf_restarts : - files = glob.glob(rst_dir+ '/*'+f+'*'+time+'*') - if len(files) >0: - restarts_in.append(files[0]) - if (len(restarts_in) == 0) : - print("\n try restart file names without time stamp\n") - for f in surf_restarts : - fname = rst_dir+ '/'+f - if os.path.exists(fname): - restarts_in.append(fname) - - return restarts_in - - def copy_merra2(self): - if not self.config['input']['shared']['MERRA-2']: - return - - expid = self.config['input']['shared']['expid'] - yyyymmddhh_ = str(self.config['input']['shared']['yyyymmddhh']) - yyyy_ = yyyymmddhh_[0: 4] - mm_ = yyyymmddhh_[4: 6] - dd_ = yyyymmddhh_[6: 8] - hh_ = yyyymmddhh_[8:10] - - suffix = yyyymmddhh_[0:8]+'_'+ hh_ + 'z.bin' - merra_2_rst_dir = MERRA2_RST_BASE + expid +'/rs/Y'+yyyy_ +'/M'+mm_+'/' - rst_dir = self.config['input']['shared']['rst_dir'] + '/' - os.makedirs(rst_dir, exist_ok = True) - print(' Copy MERRA-2 surface restarts \n from \n ' + merra_2_rst_dir + '\n to\n '+ rst_dir +'\n') - - surfin = [ merra_2_rst_dir + expid+'.lake_internal_rst.' + suffix, - merra_2_rst_dir + expid+'.landice_internal_rst.' + suffix, - merra_2_rst_dir + expid+'.saltwater_internal_rst.'+ suffix] - bin2nc_yaml = ['bin2nc_merra2_lake.yaml', 'bin2nc_merra2_landice.yaml','bin2nc_merra2_salt.yaml'] - bin_path = os.path.dirname(os.path.realpath(__file__)) - for (f,yf) in zip(surfin, bin2nc_yaml): - fname = os.path.basename(f) - dest = rst_dir + '/'+fname - print("Copy file "+f +" to " + rst_dir) - shutil.copy(f, dest) - ncdest = dest.replace('z.bin', 'z.nc4') - yaml_file = bin_path + '/'+yf - print('Convert bin to nc4:' + dest + ' to \n' + ncdest + '\n') - bin2nc(dest, ncdest, yaml_file) - os.remove(dest) - -if __name__ == '__main__' : - lls = lake_landice_saltwater(params_file='remap_params.yaml') - lls.remap() - lls.remove_geosit() + def __init__(self, **configs): + super().__init__(**configs) + if self.config["input"]["shared"]["MERRA-2"]: + self.copy_merra2() + if self.config["input"]["shared"]["GEOS-IT"]: + self.copy_geosit() + + def remap(self): + if not self.config["output"]["surface"]["remap_water"]: + return + + restarts_in = self.find_rst() + if len(restarts_in) == 0: + return + + print("\nRemapping land, landice, saltwater.....\n") + config = self.config + cwdir = os.getcwd() + bindir = os.path.dirname(os.path.realpath(__file__)) + + in_bc_base = config["input"]["shared"]["bc_base"] + in_bc_version = config["input"]["shared"]["bc_version"] + out_bc_base = config["output"]["shared"]["bc_base"] + out_bc_version = config["output"]["shared"]["bc_version"] + + agrid = config["input"]["shared"]["agrid"] + ogrid = config["input"]["shared"]["ogrid"] + omodel = config["input"]["shared"]["omodel"] + stretch = config["input"]["shared"]["stretch"] + in_tile_file = config["input"]["surface"]["catch_tilefile"] + if not in_tile_file: + EASE_grid = config["input"]["surface"].get("EASE_grid", None) + in_geomdir = get_geomdir( + in_bc_base, + in_bc_version, + agrid=agrid, + ogrid=ogrid, + omodel=omodel, + stretch=stretch, + grid=EASE_grid, + ) + in_tile_file = glob.glob(in_geomdir + "/*.til")[0] + + agrid = config["output"]["shared"]["agrid"] + ogrid = config["output"]["shared"]["ogrid"] + omodel = config["output"]["shared"]["omodel"] + stretch = config["output"]["shared"]["stretch"] + EASE_grid = config["output"]["surface"].get("EASE_grid", None) + out_tile_file = config["output"]["surface"]["catch_tilefile"] + if not out_tile_file: + out_geomdir = get_geomdir( + out_bc_base, + out_bc_version, + agrid=agrid, + ogrid=ogrid, + omodel=omodel, + stretch=stretch, + grid=EASE_grid, + ) + out_tile_file = glob.glob(out_geomdir + "/*.til")[0] + + types = ".bin" + type_str = sp.check_output(["file", "-b", os.path.realpath(restarts_in[0])]) + type_str = str(type_str) + if "Hierarchical" in type_str: + types = ".nc4" + yyyymmddhh_ = str(config["input"]["shared"]["yyyymmddhh"]) + + label = get_label(config) + suffix = yyyymmddhh_[0:8] + "_" + yyyymmddhh_[8:10] + "z" + types + label + + out_dir = config["output"]["shared"]["out_dir"] + expid = config["output"]["shared"]["expid"] + if expid: + expid = expid + "." + else: + expid = "" + + no_remap = self.copy_without_remap( + restarts_in, in_tile_file, out_tile_file, suffix + ) + if no_remap: + return + + # Determine NPE and TIME based on resolution + # Lake-landice remapping uses serial execution but needs MPI for MAPL + NPE = 1 + + # Determine output tile count to set appropriate walltime + out_Ntile = 0 + out_bc_landdir = get_landdir( + out_bc_base, + out_bc_version, + agrid=agrid, + ogrid=ogrid, + omodel=omodel, + stretch=stretch, + grid=EASE_grid, + ) + with open(out_bc_landdir + "/clsm/catchment.def") as f: + out_Ntile = int(next(f)) + + # Set TIME based on resolution + # 30 minutes for lower resolutions, 1 hour for high resolutions + TIME = "0:30:00" + if ( + out_Ntile > 277025 + ): # larger than C360 (~277k tiles) or EASEv2_M25 (~244k tiles) + TIME = "1:00:00" + + # Get job configuration from config + account = config["slurm_pbs"]["account"] + reservation = config["slurm_pbs"]["reservation"] + partition = config["slurm_pbs"]["partition"] + qos = config["slurm_pbs"]["qos"] + + # Determine job type and node configuration + NNODE = 1 + job = "" + CONSTRAINT = "" + RESERVATION = "" + PARTITION = "" + QOS = "" + + if GEOS_SITE == "NAS": + job = "PBS" + CONSTRAINT = "cas_ait" + if qos != "": + QOS = "#PBS -q " + qos + else: + job = "SLURM" + CONSTRAINT = 'mil' + if reservation != "": + RESERVATION = "#SBATCH --reservation=" + reservation + if partition != "": + PARTITION = "#SBATCH --partition=" + partition + if qos != "": + QOS = "#SBATCH --qos=" + qos + + if not os.path.exists(out_dir): + os.makedirs(out_dir) + print("cd " + out_dir) + os.chdir(out_dir) + + InData_dir = out_dir + "/InData/" + if os.path.exists(InData_dir): + sp.call(["rm", "-rf", InData_dir]) + print("mkdir " + InData_dir) + os.makedirs(InData_dir) + + OutData_dir = out_dir + "/OutData/" + if os.path.exists(OutData_dir): + sp.call(["rm", "-rf", OutData_dir]) + print("mkdir " + OutData_dir) + os.makedirs(OutData_dir) + + saltwater_internal = "" + saltwater_import = "" + seaicethermo_internal = "" + seaicethermo_import = "" + landice = "" + lake = "" + route = "" + openwater = "" + for rst in restarts_in: + f = os.path.basename(rst) + dest = InData_dir + "/" + f + if os.path.exists(dest): + shutil.remove(dest) + print("\nCopy " + rst + " to " + dest) + shutil.copy(rst, dest) + if "saltwater_internal" in f: + saltwater_internal = f + if "saltwater_import" in f: + saltwater_import = f + if "seaicethermo_internal" in f: + seaicethermo_internal = f + if "seaicethermo_import" in f: + seaicethermo_import = f + if "landice" in f: + landice = f + if "lake" in f: + lake = f + if "roue" in f: + route = f + if "openwater" in f: + openwater = f + + in_til = InData_dir + "/" + os.path.basename(in_tile_file) + out_til = OutData_dir + "/" + os.path.basename(out_tile_file) + + if os.path.exists(in_til): + shutil.remove(in_til) + if os.path.exists(out_til): + shutil.remove(out_til) + print("\n Copy " + in_tile_file + " to " + in_til) + shutil.copy(in_tile_file, in_til) + print("\n Copy " + out_tile_file + " to " + out_til) + shutil.copy(out_tile_file, out_til) + + exe = bindir + "/mk_LakeLandiceSaltRestarts.x " + zoom = config["input"]["surface"]["zoom"] + if zoom is None: + zoom = get_zoom(config) + + log_name = out_dir + "/remap_lake_landice_saltwater_log" + job_name = "remap_lake_landice_saltwater" + + # Build list of commands to execute + commands = [] + + if saltwater_internal: + cmd = ( + bindir + + "/esma_mpirun -np 1 " + + exe + + out_til + + " " + + in_til + + " InData/" + + saltwater_internal + + " 0 " + + str(zoom) + ) + commands.append(cmd) + + # split Saltwater Internal + # NOTE: split_saltwater==True means that the input restarts are already split. + # So we do not split them again. + if not config["input"]["surface"]["split_saltwater"]: + commands.append("echo 'Splitting Saltwater Internal...'") + cmd = ( + bindir + + "/SaltIntSplitter.x " + + out_til + + " " + + "OutData/" + + saltwater_internal + ) + commands.append(cmd) + + # We can now *remove* the unsplit saltwater internal restart to avoid confusion + commands.append("set unsplit_file = OutData/" + saltwater_internal) + commands.append("if ( -e $unsplit_file ) then") + commands.append( + " echo 'Removing unsplit saltwater internal restart: '$unsplit_file" + ) + commands.append(" /bin/rm -f $unsplit_file") + commands.append("endif") + + if saltwater_import: + cmd = ( + bindir + + "/esma_mpirun -np 1 " + + exe + + out_til + + " " + + in_til + + " InData/" + + saltwater_import + + " 0 " + + str(zoom) + ) + commands.append(cmd) + + # split Saltwater Import + # NOTE: split_saltwater==True means that the input restarts are already split. + # So we do not split them again. + if not config["input"]["surface"]["split_saltwater"]: + commands.append("echo 'Splitting Saltwater Import...'") + cmd = ( + bindir + + "/SaltIntSplitter.x " + + out_til + + " " + + "OutData/" + + saltwater_import + ) + commands.append(cmd) + + if openwater: + cmd = ( + bindir + + "/esma_mpirun -np 1 " + + exe + + out_til + + " " + + in_til + + " InData/" + + openwater + + " 0 " + + str(zoom) + ) + commands.append(cmd) + + if seaicethermo_internal: + cmd = ( + bindir + + "/esma_mpirun -np 1 " + + exe + + out_til + + " " + + in_til + + " InData/" + + seaicethermo_internal + + " 0 " + + str(zoom) + ) + commands.append(cmd) + + if seaicethermo_import: + cmd = ( + bindir + + "/esma_mpirun -np 1 " + + exe + + out_til + + " " + + in_til + + " InData/" + + seaicethermo_import + + " 0 " + + str(zoom) + ) + commands.append(cmd) + + if lake: + cmd = ( + bindir + + "/esma_mpirun -np 1 " + + exe + + out_til + + " " + + in_til + + " InData/" + + lake + + " 19 " + + str(zoom) + ) + commands.append(cmd) + + if landice: + cmd = ( + bindir + + "/esma_mpirun -np 1 " + + exe + + out_til + + " " + + in_til + + " InData/" + + landice + + " 20 " + + str(zoom) + ) + commands.append(cmd) + + if route: + cmd = bindir + "/mk_RouteRestarts.x " + out_til + " " + yyyymmddhh_[0:6] + commands.append(cmd) + + # Create job script with all commands + commands_str = "\n".join(commands) + + remap_template = ( + job_directive[job] + + """ +source {Bin}/g5_modules +limit stacksize unlimited + +cd {out_dir} + +set mpi_type = "openmpi" +if ($?I_MPI_ROOT ) then + set mpi_type = "intel" +endif + +if ($mpi_type =~ "openmpi") then +setenv OMPI_MCA_shmem_mmap_enable_nfs_warning 0 +setenv OMPI_MCA_mpi_preconnect_all 1 +setenv OMPI_MCA_coll_tuned_bcast_algorithm 7 +setenv OMPI_MCA_coll_tuned_scatter_algorithm 2 +setenv OMPI_MCA_coll_tuned_reduce_scatter_algorithm 3 +setenv OMPI_MCA_coll_tuned_allreduce_algorithm 3 +setenv OMPI_MCA_coll_tuned_allgather_algorithm 4 +setenv OMPI_MCA_coll_tuned_allgatherv_algorithm 3 +setenv OMPI_MCA_coll_tuned_gather_algorithm 1 +setenv OMPI_MCA_coll_tuned_barrier_algorithm 0 +setenv OMPI_MCA_coll_tuned_use_dynamic_rules 1 +setenv OMPI_MCA_sharedfp "^lockedfile,individual" +else +setenv I_MPI_FABRICS ofi +setenv I_MPI_OFI_PROVIDER psm3 +setenv I_MPI_ADJUST_SCATTER 2 +setenv I_MPI_ADJUST_SCATTERV 2 +setenv I_MPI_ADJUST_GATHER 2 +setenv I_MPI_ADJUST_GATHERV 3 +setenv I_MPI_ADJUST_ALLGATHER 3 +setenv I_MPI_ADJUST_ALLGATHERV 3 +setenv I_MPI_ADJUST_ALLREDUCE 12 +setenv I_MPI_ADJUST_REDUCE 10 +setenv I_MPI_ADJUST_BCAST 11 +setenv I_MPI_ADJUST_REDUCE_SCATTER 4 +setenv I_MPI_ADJUST_BARRIER 9 +env | grep 'I_MPI\\|FI_' +endif + +# Execute remapping commands +{commands} + +""" + ) + script_name = out_dir + "/remap_lake_landice_saltwater.j" + + remap_script_content = remap_template.format( + Bin=bindir, + account=account, + out_dir=out_dir, + log_name=log_name, + job_name=job_name, + NPE=NPE, + NNODE=NNODE, + RESERVATION=RESERVATION, + QOS=QOS, + TIME=TIME, + CONSTRAINT=CONSTRAINT, + PARTITION=PARTITION, + commands=commands_str, + ) + + with open(script_name, "w") as f: + f.write(remap_script_content) + + # Determine if running in interactive mode or batch mode + interactive = None + if GEOS_SITE == "NAS": + interactive = os.getenv("PBS_JOBID", default=None) + else: + interactive = os.getenv("SLURM_JOB_ID", default=None) + + if interactive: + print("interactive mode\n") + subprocess.call(["chmod", "755", script_name]) + print(script_name + " 1>" + log_name + " 2>&1") + os.system(script_name + " 1>" + log_name + " 2>&1") + elif GEOS_SITE == "NAS": + print("qsub -W block=true " + script_name + "\n") + subprocess.call(["qsub", "-W", "block=true", script_name]) + else: + print("sbatch -W " + script_name + "\n") + subprocess.call(["sbatch", "-W", script_name]) + + suffix = "_rst." + suffix + for out_rst in glob.glob("OutData/*_rst*"): + filename = ( + expid + + os.path.basename(out_rst).split("_rst")[0].split(".")[-1] + + suffix + ) + print("\n Move " + out_rst + " to " + out_dir + "/" + filename) + shutil.move(out_rst, out_dir + "/" + filename) + print("cd " + cwdir) + os.chdir(cwdir) + + if self.config["input"]["shared"]["MERRA-2"]: + self.remove_merra2() + + def find_rst(self): + surf_restarts = [ + "route_internal_rst", + "lake_internal_rst", + "landice_internal_rst", + "openwater_internal_rst", + "saltwater_internal_rst", + "saltwater_import_rst", + "seaicethermo_internal_rst", + "seaicethermo_import_rst", + ] + + rst_dir = self.config["input"]["shared"]["rst_dir"] + yyyymmddhh_ = str(self.config["input"]["shared"]["yyyymmddhh"]) + time = yyyymmddhh_[0:8] + "_" + yyyymmddhh_[8:10] + restarts_in = [] + for f in surf_restarts: + files = glob.glob(rst_dir + "/*" + f + "*" + time + "*") + if len(files) > 0: + restarts_in.append(files[0]) + if len(restarts_in) == 0: + print("\n try restart file names without time stamp\n") + for f in surf_restarts: + fname = rst_dir + "/" + f + if os.path.exists(fname): + restarts_in.append(fname) + + return restarts_in + + def copy_merra2(self): + if not self.config["input"]["shared"]["MERRA-2"]: + return + + expid = self.config["input"]["shared"]["expid"] + yyyymmddhh_ = str(self.config["input"]["shared"]["yyyymmddhh"]) + yyyy_ = yyyymmddhh_[0:4] + mm_ = yyyymmddhh_[4:6] + dd_ = yyyymmddhh_[6:8] + hh_ = yyyymmddhh_[8:10] + + suffix = yyyymmddhh_[0:8] + "_" + hh_ + "z.bin" + merra_2_rst_dir = MERRA2_RST_BASE + expid + "/rs/Y" + yyyy_ + "/M" + mm_ + "/" + rst_dir = self.config["input"]["shared"]["rst_dir"] + "/" + os.makedirs(rst_dir, exist_ok=True) + print( + " Copy MERRA-2 surface restarts \n from \n " + + merra_2_rst_dir + + "\n to\n " + + rst_dir + + "\n" + ) + + surfin = [ + merra_2_rst_dir + expid + ".lake_internal_rst." + suffix, + merra_2_rst_dir + expid + ".landice_internal_rst." + suffix, + merra_2_rst_dir + expid + ".saltwater_internal_rst." + suffix, + ] + bin2nc_yaml = [ + "bin2nc_merra2_lake.yaml", + "bin2nc_merra2_landice.yaml", + "bin2nc_merra2_salt.yaml", + ] + bin_path = os.path.dirname(os.path.realpath(__file__)) + for f, yf in zip(surfin, bin2nc_yaml): + fname = os.path.basename(f) + dest = rst_dir + "/" + fname + print("Copy file " + f + " to " + rst_dir) + shutil.copy(f, dest) + ncdest = dest.replace("z.bin", "z.nc4") + yaml_file = bin_path + "/" + yf + print("Convert bin to nc4:" + dest + " to \n" + ncdest + "\n") + bin2nc(dest, ncdest, yaml_file) + os.remove(dest) + + +if __name__ == "__main__": + lls = lake_landice_saltwater(params_file="remap_params.yaml") + lls.remap() + lls.remove_geosit() diff --git a/pre/remap_restart/remap_params.tpl b/pre/remap_restart/remap_params.tpl index cc76147b..a67c6e12 100644 --- a/pre/remap_restart/remap_params.tpl +++ b/pre/remap_restart/remap_params.tpl @@ -22,6 +22,7 @@ input: rst_dir: yyyymmddhh: surface: + split_saltwater: true zoom: wemin: # it supports three models: catch, catchcnclm40, catchcnclm51 @@ -66,3 +67,4 @@ slurm_pbs: account: qos: '' partition: '' + reservation: '' diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index 786bc1cb..e11f8909 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -186,7 +186,7 @@ def remap(self): if (qos != ''): QOS = "#SBATCH --qos="+qos - CONSTRAINT = '"[cas|mil]"' + CONSTRAINT = 'mil' # We need to create an input.nml file which is different if we are running stretched grid # If we are running stretched grid, we need to pass in the target lon+lat and stretch factor diff --git a/pre/remap_restart/tests/amip_c180Toc90.yaml b/pre/remap_restart/tests/amip_c180Toc90.yaml index 36278f3f..951ddea5 100644 --- a/pre/remap_restart/tests/amip_c180Toc90.yaml +++ b/pre/remap_restart/tests/amip_c180Toc90.yaml @@ -21,6 +21,7 @@ input: rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/amip_c180Toc90/inputs/ yyyymmddhh: '1990062621' surface: + split_saltwater: true zoom: '2' wemin: '13' # it supports three models: catch, catchcnclm40, catchcnclm45 @@ -62,3 +63,4 @@ slurm_pbs: account: qos: debug partition: '' + reservation: '' diff --git a/pre/remap_restart/tests/c180Toc360.yaml b/pre/remap_restart/tests/c180Toc360.yaml index 7c3ee8b5..53b34b28 100644 --- a/pre/remap_restart/tests/c180Toc360.yaml +++ b/pre/remap_restart/tests/c180Toc360.yaml @@ -21,6 +21,7 @@ input: rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/c180Toc360/inputs/ yyyymmddhh: '1985050921' surface: + split_saltwater: true zoom: '2' wemin: '13' # it supports three models: catch, catchcnclm40, catchcnclm45 @@ -62,3 +63,4 @@ slurm_pbs: account: qos: debug partition: '' + reservation: '' diff --git a/pre/remap_restart/tests/c24Toc12.yaml b/pre/remap_restart/tests/c24Toc12.yaml index 6ce7a3e4..ef58a287 100644 --- a/pre/remap_restart/tests/c24Toc12.yaml +++ b/pre/remap_restart/tests/c24Toc12.yaml @@ -21,6 +21,7 @@ input: rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/c24Toc12/inputs yyyymmddhh: '2000041421' surface: + split_saltwater: true zoom: '1' wemin: '13' # it supports three models: catch, catchcnclm40, catchcnclm45 @@ -62,3 +63,4 @@ slurm_pbs: account: qos: debug partition: '' + reservation: '' diff --git a/pre/remap_restart/tests/c360Toc24.yaml b/pre/remap_restart/tests/c360Toc24.yaml index 7e6eee2a..de7f7fa4 100644 --- a/pre/remap_restart/tests/c360Toc24.yaml +++ b/pre/remap_restart/tests/c360Toc24.yaml @@ -21,6 +21,7 @@ input: rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/c360Toc24/inputs yyyymmddhh: '2022010121' surface: + split_saltwater: true zoom: '4' wemin: '13' # it supports three models: catch, catchcnclm40, catchcnclm45 @@ -62,3 +63,4 @@ slurm_pbs: account: qos: debug partition: '' + reservation: '' diff --git a/pre/remap_restart/tests/f522Toc360.yaml b/pre/remap_restart/tests/f522Toc360.yaml index 6f133206..6bca0167 100644 --- a/pre/remap_restart/tests/f522Toc360.yaml +++ b/pre/remap_restart/tests/f522Toc360.yaml @@ -21,6 +21,7 @@ input: rst_dir: /discover/nobackup/projects/gmao/SIteam/Remapping_Test_Cases/f522Toc360/inputs/ yyyymmddhh: '2019061421' surface: + split_saltwater: true zoom: '8' wemin: '26' catch_model: catch @@ -58,3 +59,4 @@ slurm_pbs: account: qos: debug partition: '' + reservation: '' diff --git a/pre/remap_restart/tests/test_remap_restarts.py b/pre/remap_restart/tests/test_remap_restarts.py index 6d64a746..fa6a3e6e 100755 --- a/pre/remap_restart/tests/test_remap_restarts.py +++ b/pre/remap_restart/tests/test_remap_restarts.py @@ -31,7 +31,7 @@ def compare(base, result): basedir = os.environ['BASEDIR'] NCCMP = basedir+'/Linux/bin/nccmp' for b, r in zip(bases, results): - cmd = NCCMP + ' -dmgfs '+ b + ' ' + r + cmd = NCCMP + ' -dmgs '+ b + ' ' + r print(cmd) p = sp.Popen(shlex.split(cmd), stdout=subprocess.PIPE) (out, err) = p.communicate()