Skip to content

revised RESTART options; fix string matching for EASE tile file #160

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gmao-rreichle merged 16 commits into from
Mar 20, 2026
Merged

revised RESTART options; fix string matching for EASE tile file #160

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
68d08a1
exclude tile2pfaf nc4 file which is not a tile file
weiyuan-jiang Feb 11, 2026
97e8864
remove extra line in ldas.py
weiyuan-jiang Feb 11, 2026
0618cfc
fix bug to find extention and reorg. assert if domain is wrong
weiyuan-jiang Feb 18, 2026
5cae2f1
fix indent
weiyuan-jiang Feb 18, 2026
fc0e9ed
check bcs_path and resolution when RESTART = 1
weiyuan-jiang Mar 2, 2026
2e406ec
remove restart=0, add restart=3
weiyuan-jiang Mar 2, 2026
9ba7cac
rename new output file to "BCS_info.txt"
gmao-rreichle Mar 4, 2026
a65b644
edited comments about RESTART options (ldas.py, setup_utils.py)
gmao-rreichle Mar 4, 2026
2407a13
minimal edit of comment about RESTART (ldas.py)
gmao-rreichle Mar 4, 2026
d4cf955
exclude special tile file for route
weiyuan-jiang Mar 6, 2026
7e54ab6
fixed typo in variable name; edited comments (ldas.py)
gmao-rreichle Mar 12, 2026
e29c918
fixed variable name typo; edited comments (ldas.py, setup_utils.py)
gmao-rreichle Mar 13, 2026
5747db0
updated CHANGELOG.md
gmao-rreichle Mar 13, 2026
4772bc8
Merge branch 'develop' into feature/wjiang/exclude_pfaf_nc4
gmao-rreichle Mar 13, 2026
3de973b
remove unnecessary check
weiyuan-jiang Mar 16, 2026
5e519c4
correct output for BCS_info.txt
weiyuan-jiang Mar 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
160 changes: 63 additions & 97 deletions GEOSldas_App/ldas.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,18 +129,13 @@ def __init__(self, cmdLineArgs):
if 'RESTART' not in self.ExeInputs :
self.ExeInputs['RESTART'] = "1"

if self.ExeInputs['RESTART'].isdigit() :
if int(self.ExeInputs['RESTART']) ==0 :
self.ExeInputs['RESTART_ID'] = 'None'
self.ExeInputs['RESTART_DOMAIN'] = 'None'
self.ExeInputs['RESTART_PATH'] = 'None'
else:
if self.ExeInputs['RESTART'] =='G' :
self.ExeInputs['RESTART_DOMAIN'] = 'None'
else:
self.ExeInputs['RESTART_ID'] = 'None'
self.ExeInputs['RESTART_DOMAIN'] = 'None'
self.ExeInputs['RESTART_PATH'] = 'None'
RESTART_str = str(self.ExeInputs['RESTART'])
assert RESTART_str in ["1", "2", "3", "M"], "Only support restart = 1,2,3, M"

if RESTART_str == 'M':
self.ExeInputs['RESTART_ID'] = 'None'
self.ExeInputs['RESTART_DOMAIN'] = 'None'
self.ExeInputs['RESTART_PATH'] = 'None'

### check if ldas is coupled to adas; if so, set/overwrite input parameters accordingly
if self.ladas_cpl > 0 :
Expand Down Expand Up @@ -279,15 +274,36 @@ def __init__(self, cmdLineArgs):
inpdir_ = self.bcs_dir_land
inpgeom_ = self.bcs_dir_geom

if self.ExeInputs['RESTART'] == '1' :
inp_ = self.ExeInputs['RESTART_PATH']+'/'.join([self.ExeInputs['RESTART_ID'],'output',

# find restart files and tile files (if necessary)

inp_ = self.ExeInputs['RESTART_PATH']+'/'.join([self.ExeInputs['RESTART_ID'],'output',
self.ExeInputs['RESTART_DOMAIN'], 'rc_out/'])
if RESTART_str == '1' :
inpdir_ = inp_
inpgeom_ = inp_
BCS_txt = glob.glob(inp_ + 'BCS.txt')
if len(BCS_txt)== 0:
print("Warning: There is no BCS.txt in rc_out. The BCS in the restart directory may not be consistent with " + self.ExeInputs['BCS_PATH'])
elif len(BCS_txt)== 1:
BCS_tmp = parseInputFile(BCS_txt[0])
assert self.ExeInputs['BCS_PATH'] == BCS_tmp['BCS_PATH'], "BCS_PATH does not match " + BCS_tmp['BCS_PATH'] + " in restart directoy"
assert self.ExeInputs['BCS_RESOLUTION'] == BCS_tmp['BCS_RESOLUTION'], "BCS resolution or gridname should be the same for RESTART =1"

txt_tile = glob.glob(inp_ + '*.domain')
if len(txt_tile) > 0:
domain_ = '.domain'
inpdir_ = inp_
inpgeom_ = inp_

elif RESTART_str == '2':
txt_tile = glob.glob(inp_ + '*.domain')
assert len(txt_tile) == 0, "RESTART = 2 should restart from global domain"
in_tilefiles_ = glob.glob(inp_+'MAPL_*.til')
if len(in_tilefiles_) == 0 :
nc4_tmp = glob.glob(inp_+'/*.nc4')
in_tilesfile_ = [ item_ for item_ in nc4_tmp if 'tile2pfaf' not in item_ ]
if len(in_tilefiles_) == 0 :
in_tilefiles_ = glob.glob(inp_+'/*.til')
self.in_tilefile =os.path.realpath(in_tilefiles_[0])

inpdir_ = os.path.realpath(inpdir_)+'/'
inpgeom_ = os.path.realpath(inpgeom_)+'/'

Expand All @@ -296,7 +312,8 @@ def __init__(self, cmdLineArgs):
if 'MAPL_' in os.path.basename(f):
txt_tile = [f]
break
nc4_tile = glob.glob(inpgeom_ + '*.nc4' + domain_)
nc4_tmp = glob.glob(inpgeom_ + '*.nc4' + domain_)
nc4_tile = [ item_ for item_ in nc4_tmp if 'tile2pfaf' not in item_ ]
if tile_file_format.upper() == 'TXT' : self.ExeInputs['TILING_FILE'] = txt_tile[0]
if tile_file_format.upper() == 'DEFAULT' : self.ExeInputs['TILING_FILE'] = (txt_tile+nc4_tile)[-1]

Expand All @@ -316,7 +333,7 @@ def __init__(self, cmdLineArgs):
tmptile = os.path.realpath(self.ExeInputs['TILING_FILE'])
extension = os.path.splitext(tmptile)[1]
if extension == '.domain':
extension = os.path.splitext(tmptile)[0]
extension = os.path.splitext(os.path.splitext(tmptile)[0])[1]
gridname_ =''
if extension == '.til':
gridname_ = linecache.getline(tmptile, 3).strip()
Expand Down Expand Up @@ -346,7 +363,8 @@ def __init__(self, cmdLineArgs):

for key,val in _domain_dic.items() :
if key in self.ExeInputs :
_domain_dic[key]= self.ExeInputs[key]
_domain_dic[key]= self.ExeInputs[key]
assert RESTART_str != 2, "Please comment out " + key + " event you set it global"
self.domain_def = tempfile.NamedTemporaryFile(mode='w', delete=False)
self.domain_def.write('&domain_inputs\n')
for key,val in _domain_dic.items() :
Expand All @@ -359,22 +377,7 @@ def __init__(self, cmdLineArgs):
self.domain_def.write('/\n')
self.domain_def.close()

# find restart files and tile files (if necessary)
RESTART_str = str(self.ExeInputs['RESTART'])

if RESTART_str == '2':
inpdir=self.ExeInputs['RESTART_PATH']+self.ExeInputs['RESTART_ID']+'/input/'
in_tilefiles_ = glob.glob(inpdir+'*tile.data')
if len(in_tilefiles_) == 0 :
inpdir=self.ExeInputs['RESTART_PATH']+self.ExeInputs['RESTART_ID']+'/output/'+self.ExeInputs['RESTART_DOMAIN']+'/rc_out/'
in_tilefiles_ = glob.glob(inpdir+'MAPL_*.til')
if len(in_tilefiles_) == 0 :
in_tilefiles_ = glob.glob(inpdir+'/*.til')
if len(in_tilefiles_) == 0 :
in_tilefiles_ = glob.glob(inpdir+'/*.nc4')
self.in_tilefile =os.path.realpath(in_tilefiles_[0])

if RESTART_str in ['1', '2']:
if RESTART_str in ['1','2','3']:
y4m2='Y%4d/M%02d' % (self.begDates[0].year, self.begDates[0].month)
y4m2d2_h2m2='%4d%02d%02d_%02d%02d' % (self.begDates[0].year, self.begDates[0].month,
self.begDates[0].day,self.begDates[0].hour,self.begDates[0].minute)
Expand All @@ -395,24 +398,6 @@ def __init__(self, cmdLineArgs):
landiceRstFile=self.in_rstdir+'/'+tmpFile
assert os.path.isfile(landiceRstFile), 'landice_internal_rst file [%s] does not exist!' %(landiceRstFile)

if RESTART_str == '0':
assert ( self.with_land and not self.with_landice), "RESTART = 0 is only for land"
if (self.catch == 'catch'):
self.in_rstdir = '/discover/nobackup/projects/gmao/ssd/land/l_data/LandRestarts_for_Regridding' \
'/Catch/M09/20170101/' #catch_internal_rst
self.in_tilefile = '/discover/nobackup/projects/gmao/ssd/land/l_data/geos5/bcs/CLSM_params' \
'/mkCatchParam_SMAP_L4SM_v002/SMAP_EASEv2_M09/SMAP_EASEv2_M09_3856x1624.til'
elif (self.catch == 'catchcnclm40'):
self.in_rstdir = '/discover/nobackup/projects/gmao/ssd/land/l_data/LandRestarts_for_Regridding' \
'/CatchCN/M36/20150301_0000/' #catchcnclm40_internal_dummy
self.in_tilefile = '/discover/nobackup/projects/gmao/bcs_shared/legacy_bcs/Heracles-NL/SMAP_EASEv2_M36/SMAP_EASEv2_M36_964x406.til'
elif (self.catch == 'catchcnclm45'):
self.in_rstdir = '/discover/nobackup/projects/gmao/ssd/land/l_data/LandRestarts_for_Regridding' \
'/CatchCN/M36/19800101_0000/' #catchcnclm45_internal_dummy
self.in_tilefile = '/discover/nobackup/projects/gmao/bcs_shared/legacy_bcs/Icarus-NLv3/Icarus-NLv3_EASE/SMAP_EASEv2_M36/SMAP_EASEv2_M36_964x406.til'
else:
sys.exit('need to provide at least dummy files')

# DEAL WITH mwRTM input from exec
self.assim = True if self.ExeInputs.get('LAND_ASSIM', 'NO').upper() == 'YES' and self.with_land else False
# verify mwrtm file
Expand Down Expand Up @@ -472,7 +457,7 @@ def __init__(self, cmdLineArgs):
self.scratchdir = self.expdir + '/scratch'
self.blddirLn = self.expdir + '/build'
self.out_path = self.outdir + '/'+self.ExeInputs['EXP_DOMAIN']
self.bcsdir = self.outdir + '/'+self.ExeInputs['EXP_DOMAIN']+'/rc_out/'
self.rc_out = self.outdir + '/'+self.ExeInputs['EXP_DOMAIN']+'/rc_out/'
self.rstdir = self.outdir + '/'+self.ExeInputs['EXP_DOMAIN']+'/rs/'
self.exefyl = self.blddirLn + exefyl

Expand All @@ -493,8 +478,6 @@ def _verifyExeInputs(self):
ExeInputs = self.ExeInputs
#) verify keys
option = '1'
if (ExeInputs['RESTART'] == 'G' or ExeInputs['RESTART'] == '0'):
option = '0'

rqdExeInpKeys = getExeKeys(option)
for key in rqdExeInpKeys:
Expand Down Expand Up @@ -543,11 +526,6 @@ def _calculateJobSegments(self):
'%Y%m%d %H%M%S'
)
)
if self.ExeInputs['RESTART'].isdigit() :
if int(self.ExeInputs['RESTART']) == 0 :
print ("No restart file (cold restart): Forcing start date to January 1, 0z")
year = self.begDates[0].year
self.begDates[0]=datetime(year =year,month=1,day =1,hour =0, minute= 0,second= 0)

assert self.endDates[0]>self.begDates[0], \
'END_DATE <= BEG_DATE'
Expand Down Expand Up @@ -689,13 +667,13 @@ def createLnRstBc(self) :
# update tile file
tile= self.ExeInputs['TILING_FILE']
short_tile= os.path.basename(self.ExeInputs['TILING_FILE'])
newtile = self.bcsdir+'/'+short_tile
newtile = self.rc_out+'/'+short_tile
shutil.copy(tile, newtile)
tile=newtile
# if three extra lines exist, remove them and save it to inputdir

print ('\nCorrect the tile file if it is an old EASE tile format... \n')
EASEtile=self.bcsdir+'/MAPL_'+short_tile
EASEtile=self.rc_out+'/MAPL_'+short_tile
cmd = self.bindir + '/preprocess_ldas.x correctease '+ tile + ' '+ EASEtile
print ("cmd: " + cmd)

Expand Down Expand Up @@ -762,8 +740,8 @@ def createLnRstBc(self) :
bcs += [self.ExeInputs['VEGOPACITY_FILE']]
bcstmp=[]
for bcf in bcs :
shutil.copy(bcf, self.bcsdir+'/')
bcstmp=bcstmp+[self.bcsdir+'/'+os.path.basename(bcf)]
shutil.copy(bcf, self.rc_out+'/')
bcstmp=bcstmp+[self.rc_out+'/'+os.path.basename(bcf)]
bcs=bcstmp

if self.isZoomIn:
Expand Down Expand Up @@ -801,13 +779,6 @@ def createLnRstBc(self) :

myRstDir = self.inpdir + '/restart/'

rstpath = self.ExeInputs['RESTART_PATH']+ \
self.ExeInputs['RESTART_ID'] + \
'/output/'+self.ExeInputs['RESTART_DOMAIN']+'/rs/'
rcoutpath = self.ExeInputs['RESTART_PATH']+ \
self.ExeInputs['RESTART_ID'] + \
'/output/'+self.ExeInputs['RESTART_DOMAIN']+'/rc_out/'

# pass into remap_config_ldas
exp_id = self.ExeInputs['EXP_ID']
RESTART_str = str(self.ExeInputs['RESTART'])
Expand All @@ -816,6 +787,7 @@ def createLnRstBc(self) :
rstid = self.ExeInputs['RESTART_ID']
rstdomain = self.ExeInputs['RESTART_DOMAIN']
rstpath0 = self.ExeInputs['RESTART_PATH']
rstpath = rstpath0 + rstid + '/output/'+ rstdomain + '/rs/'

# just copy the landassim pert seed if it exists
for iens in range(self.nens) :
Expand All @@ -829,7 +801,7 @@ def createLnRstBc(self) :
self.has_landassim_seed = True
mk_outdir = self.exphome+'/'+exp_id+'/mk_restarts/'

if (RESTART_str != '1' and (self.with_land or self.with_landice)):
if (RESTART_str in ['2', 'M'] and (self.with_land or self.with_landice)):
bcs_path = self.ExeInputs['BCS_PATH']
while bcs_path[-1] == '/' : bcs_path = bcs_path[0:-1]
bc_base = os.path.dirname(bcs_path)
Expand All @@ -844,7 +816,7 @@ def createLnRstBc(self) :
config['input']['surface']['catch_tilefile'] = self.in_tilefile
config['input']['shared']['expid'] = self.ExeInputs['RESTART_ID']
config['input']['shared']['yyyymmddhh'] = YYYYMMDDHH
if RESTART_str != 'M':
if RESTART_str == '2':
config['input']['shared']['rst_dir'] = self.in_rstdir
config['input']['surface']['wemin'] = wemin_in
config['input']['surface']['catch_model'] = self.catch
Expand Down Expand Up @@ -911,24 +883,18 @@ def createLnRstBc(self) :
catchRstFile = ''
vegdynRstFile = ''
pertRstFile = ''
print ("restart: " + self.ExeInputs['RESTART'])
print ("restart: " + RESTART_str)

if self.with_land :
if self.ExeInputs['RESTART'].isdigit() :

if int(self.ExeInputs['RESTART']) == 0 or int(self.ExeInputs['RESTART']) == 2 :
if RESTART_str in ['2', 'M']: # remap_restart has generated the file in mk_restarts directory
vegdynRstFile = glob.glob(self.bcs_dir_land + 'vegdyn_*.dat')[0]
catchRstFile = glob.glob(self.exphome+'/'+exp_id+'/mk_restarts/*'+self.catch+'_internal_rst.'+YYYYMMDD+'*')[0]
if RESTART_str in ['1', '3']:
catchRstFile = rstpath+ensdir +'/'+ y4m2+'/'+self.ExeInputs['RESTART_ID']+'.'+self.catch+'_internal_rst.'+y4m2d2_h2m2
vegdynRstFile= rstpath+ensdir +'/'+self.ExeInputs['RESTART_ID']+ '.vegdyn_internal_rst'
if RESTART_str == '3':
vegdynRstFile = glob.glob(self.bcs_dir_land + 'vegdyn_*.dat')[0]
catchRstFile = glob.glob(self.exphome+'/'+exp_id+'/mk_restarts/*'+self.catch+'_internal_rst.'+YYYYMMDD+'*')[0]
else : # RESTART == 1
catchRstFile = rstpath+ensdir +'/'+ y4m2+'/'+self.ExeInputs['RESTART_ID']+'.'+self.catch+'_internal_rst.'+y4m2d2_h2m2
vegdynRstFile= rstpath+ensdir +'/'+self.ExeInputs['RESTART_ID']+ '.vegdyn_internal_rst'
if not os.path.isfile(vegdynRstFile): # no vegdyn restart from LDASsa
if not os.path.isfile(vegdynRstFile0):
vegdynRstFile = glob.glob(self.bcs_dir_land + 'vegdyn_*.dat')[0]
else :
vegdynRstFile = glob.glob(self.bcs_dir_land + 'vegdyn_*.dat')[0]
if self.with_land:
catchRstFile = glob.glob(self.exphome+'/'+exp_id+'/mk_restarts/*'+self.catch+'_internal_rst.'+YYYYMMDD+'*')[0]

# catchment restart file
if os.path.isfile(catchRstFile) :
Expand Down Expand Up @@ -968,11 +934,9 @@ def createLnRstBc(self) :

landiceRstFile = ''
if self.with_landice :
if RESTART_str == '0' :
exit("RESTART=0 not supported for landice tiles. Please use RESTART=M, 1, or 2")
if RESTART_str == '1' :
if RESTART_str in ['1', '3'] :
landiceRstFile = rstpath+ensdir +'/'+ y4m2+'/'+self.ExeInputs['RESTART_ID']+'.'+'landice_internal_rst.'+y4m2d2_h2m2
if RESTART_str == '2' or RESTART_str == 'M':
if RESTART_str in ['2', 'M']:
landiceRstFile = glob.glob(self.exphome+'/'+exp_id+'/mk_restarts/*'+'landice_internal_rst.'+YYYYMMDD+'*')[0]

if os.path.isfile(landiceRstFile) :
Expand Down Expand Up @@ -1010,13 +974,13 @@ def createLnRstBc(self) :
os.symlink(pertRstFile, myPertRst)

# catch_param restar file
catch_param_file = self.bcsdir+'/'+ y4m2+'/'+self.ExeInputs['EXP_ID']+'.ldas_catparam.'+y4m2d2_h2m2+'z.bin'
catch_param_file = self.rc_out+'/'+ y4m2+'/'+self.ExeInputs['EXP_ID']+'.ldas_catparam.'+y4m2d2_h2m2+'z.bin'
if self.with_land:
assert os.path.isfile(catch_param_file), "need catch_param file %s" % catch_param_file

if self.has_mwrtm :
mwRTMRstFile = self.mwrtm_file
mwRTMLocal = self.bcsdir+'/'+ y4m2+'/'+self.ExeInputs['EXP_ID']+'.ldas_mwRTMparam.'+y4m2d2_h2m2+'z.nc4'
mwRTMLocal = self.rc_out+'/'+ y4m2+'/'+self.ExeInputs['EXP_ID']+'.ldas_mwRTMparam.'+y4m2d2_h2m2+'z.nc4'
if self.isZoomIn :
print ("Creating the local mwRTM restart file... \n")
cmd= self.bindir +'/preprocess_ldas.x zoomin_mwrtmrst '+ mwRTMRstFile +' ' + mwRTMLocal + ' '+ tmp_f2g_file.name
Expand All @@ -1030,11 +994,13 @@ def createLnRstBc(self) :
mymwRTMRst = myRstDir+'/mwrtm_param_rst'
os.symlink(mwRTMRstFile, mymwRTMRst)

#create BCS.txt
with open(self.rc_out+'/BCS.txt','wt') as fout :
fout.write("BCS_PATH: " + self.ExeInputs['BCS_PATH'])
fout.write("BCS_RESOLUTION: " + self.ExeInputs['BCS_RESOLUTION'])
# update 'restart_path' to use relative path from outdir
print ("Updating restart path...")
self.ExeInputs['RESTART_PATH'] = myRstDir
#if os.path.isfile(tmp_f2g_file.name):
# os.remove(tmp_f2g_file.name)
status = True
return status

Expand Down
Loading
Loading