diff --git a/daily_tiler.py b/daily_tiler.py index 9793739..2a95e9b 100644 --- a/daily_tiler.py +++ b/daily_tiler.py @@ -31,7 +31,6 @@ def main(rawargs=None): parser = argparse.ArgumentParser(description="Daily Tiler") parser.add_argument('file_list',type=str) parser.add_argument('output_dest', type=str) - parser.add_argument('--dynamic', action='store_true') args = parser.parse_args(rawargs) files = open(args.file_list,'r').readlines() @@ -42,10 +41,7 @@ def main(rawargs=None): count = 0 success=False - if args.dynamic: - subprocess.call(f'python /beegfs/scratch/brodrick/emit/MMGIS/auxiliary/gdal2customtiles/gdal2tiles_3.5.2.py --dem {fi} {args.output_dest} -r near-composite -z 5-11 --srcnodata=-9999 --processes=40 && echo {fi} >> {os.path.splitext(args.file_list)[0] + "_completed.txt"}',shell=True) - else: - subprocess.call(f'gdal2tiles.py {fi} {args.output_dest} -z 4-12 --srcnodata 0 --processes=40 -r antialias -x && echo {fi} >> {os.path.splitext(args.file_list)[0] + "_completed.txt"}',shell=True) + subprocess.call(f'gdal2tiles.py {fi} {args.output_dest} -z 4-12 --srcnodata 0 --processes=40 -r antialias -x && echo {fi} >> {os.path.splitext(args.file_list)[0] + "_completed.txt"}',shell=True) diff --git a/delivery_plume_tiler.py b/delivery_plume_tiler.py index 5be4340..e6adeb1 100644 --- a/delivery_plume_tiler.py +++ b/delivery_plume_tiler.py @@ -186,13 +186,14 @@ def main(input_args=None): parser = argparse.ArgumentParser(description="Delineate/colorize plume") parser.add_argument('--source_dir', type=str, default='methane_20230813') parser.add_argument('--dest_dir', type=str, default='visions_delivery') - parser.add_argument('--manual_del_dir', type=str, default='/beegfs/scratch/brodrick/methane/ch4_plumedir_scenetest/') + parser.add_argument('--manual_del_dir', type=str, default='/store/brodrick/methane/ch4_plumedir_scenetest/') parser.add_argument('--software_version', type=str, default=None) parser.add_argument('--data_version', type=str, default=None) parser.add_argument('--visions_delivery', type=int, choices=[0,1,2],default=0) parser.add_argument('--n_cores', type=int, default=1) parser.add_argument('--overwrite', action='store_true') - parser.add_argument('--previous_plume_file', type=str, default='visions_delivery/plume_list.txt') + parser.add_argument('--delivery_dir', type=str, default='visions_delivery') + parser.add_argument('--previous_plume_file', type=str, default='plume_list.txt') parser.add_argument('--loglevel', type=str, default='DEBUG', help='logging verbosity') parser.add_argument('--logfile', type=str, default=None, help='output file to write log to') args = parser.parse_args(input_args) @@ -202,7 +203,7 @@ def main(input_args=None): tile_dir = os.path.join(args.dest_dir, 'ch4_plume_tiles') - with open(args.previous_plume_file,'r') as f: + with open(os.path.join(args.delivery_dir, args.previous_plume_file),'r') as f: delivered_plume_names = f.read().splitlines() delivered_plume_ids = [f'CH4_PlumeComplex-{x.split("_")[-1]}' for x in delivered_plume_names] @@ -318,7 +319,7 @@ def main(input_args=None): with open(os.path.join(args.dest_dir, 'combined_plume_metadata.json'), 'w') as fout: fout.write(json.dumps(outdict, cls=SerialEncoder)) - subprocess.call("rsync visions_delivery/combined_plume_metadata.json brodrick@${EMIT_SCIENCE_IP}:/data/emit/mmgis/coverage/combined_plume_metadata.json",shell=True) + subprocess.call(f"rsync {args.delivery_dir}/combined_plume_metadata.json brodrick@${EMIT_SCIENCE_IP}:/data/emit/mmgis/coverage/combined_plume_metadata.json",shell=True) logging.info('Tile output') @@ -329,7 +330,7 @@ def main(input_args=None): subfeatures = [feat for _feat, feat in enumerate(all_plume_meta['features']) if _feat in match_idx and _feat in valid_plume_idx] if len(subfeatures) > 0: tile_dcid(subfeatures, outdir, args.manual_del_dir) - subprocess.call("rsync -a --info=progress2 visions_delivery/visions_ch4_tiles/ brodrick@${EMIT_SCIENCE_IP}:/data/emit/mmgis/mosaics/ch4_plume_tiles/",shell=True) + subprocess.call(f"rsync -a --info=progress2 {outdir} brodrick@${EMIT_SCIENCE_IP}:/data/emit/mmgis/mosaics/ch4_plume_tiles/",shell=True) diff --git a/run_bulk_ghg.py b/run_bulk_ghg.py index 40bfb90..427e37d 100644 --- a/run_bulk_ghg.py +++ b/run_bulk_ghg.py @@ -58,10 +58,10 @@ def main(input_args=None): launch = os.path.isfile(ch4_mf_kmz_file) is False if os.path.isfile(ch4_mf_kmz_file) is False or (args.co2 and os.path.isfile(co2_mf_kmz_file) is False): - cmd_str=f'sbatch -N 1 -c 40 -p standard --mem=180G --wrap="source /tmp/miniconda/bin/activate; conda activate isofit_env; python ghg_process.py {rdn_files[_r]} {obs_files[_r]} {loc_files[_r]} {glt_files[_r]} {l1b_bandmask_files[_r]} {l2a_mask_files[_r]} {out_files[_r]}' + cmd_str=f'sbatch -N 1 -c 64 -p standard --mem=300G --wrap="python ghg_process.py {rdn_files[_r]} {obs_files[_r]} {loc_files[_r]} {glt_files[_r]} {l1b_bandmask_files[_r]} {l2a_mask_files[_r]} {out_files[_r]}' if args.co2: - cmd_str += ' --co2' + cmd_str += ' --co2 --lut_file /store/shared/ghg/dataset_co2_full.hdf5' if state_files[_r] is not None: cmd_str += f' --state_subs {state_files[_r]}"' diff --git a/run_plumeid_ghg.py b/run_plumeid_ghg.py index 80f55c6..542f214 100644 --- a/run_plumeid_ghg.py +++ b/run_plumeid_ghg.py @@ -36,12 +36,12 @@ def main(input_args=None): plumedict = json.load(open(args.methane_metadata,'r')) - fids = [x['properties']['Scene FID'] for x in plumedict['features']] + #fids = [x['properties']['Scene FIDs'] for x in plumedict['features']] + fids = np.unique([sublist for feat in plumedict['features'] for sublist in feat['properties']['Scene FIDs']]) - un_fids = np.unique(fids) rdn_files = [] - for fid in un_fids: - glist = glob(f'/beegfs/store/emit/ops/data/acquisitions/{fid[4:12]}/{fid}/l1b/*_rdn_b0106_v01.img') + for fid in fids: + glist = glob(f'/store/emit/ops/data/acquisitions/{fid[4:12]}/{fid}/l1b/*_rdn_b0106_v01.img') if len(glist) > 0: rdn_files.append(glist[0]) else: @@ -50,7 +50,7 @@ def main(input_args=None): if rdn_files[_fid] is None: rdn_files.pop(_fid) fids.pop(_fid) - un_fids = np.unique(fids) + fids = np.unique(fids) obs_files = [x.replace('rdn','obs') for x in rdn_files] loc_files = [x.replace('rdn','loc') for x in rdn_files] @@ -61,7 +61,7 @@ def main(input_args=None): state_files = [x.replace('l1b','l2a').replace('rdn','statesubs') for x in rdn_files] state_files = [x if os.path.isfile(x) else None for x in state_files] - for fid in un_fids: + for fid in fids: date = fid[4:12] if os.path.isdir(os.path.join(args.output_dir, date)) is False: subprocess.call(f'mkdir {os.path.join(args.output_dir, date)}',shell=True) @@ -76,10 +76,10 @@ def main(input_args=None): launch = os.path.isfile(ch4_mf_kmz_file) is False if os.path.isfile(ch4_mf_kmz_file) is False or (args.co2 and os.path.isfile(co2_mf_kmz_file) is False): - cmd_str=f'sbatch -N 1 -c 40 -p standard --mem=180G --wrap="python ghg_process.py {rdn_files[_r]} {obs_files[_r]} {loc_files[_r]} {glt_files[_r]} {l1b_bandmask_files[_r]} {l2a_mask_files[_r]} {out_files[_r]}' + cmd_str=f'sbatch -N 1 -c 64 -p standard --mem=300G --wrap="python ghg_process.py {rdn_files[_r]} {obs_files[_r]} {loc_files[_r]} {glt_files[_r]} {l1b_bandmask_files[_r]} {l2a_mask_files[_r]} {out_files[_r]}' if args.co2: - cmd_str += ' --co2' + cmd_str += ' --co2 --lut_file /store/shared/ghg/dataset_co2_full.hdf5' if state_files[_r] is not None: cmd_str += f' --state_subs {state_files[_r]}"' diff --git a/run_priority.py b/run_priority.py index 878af5e..cc4018f 100644 --- a/run_priority.py +++ b/run_priority.py @@ -30,6 +30,7 @@ def main(input_args=None): parser = argparse.ArgumentParser(description="Robust MF") parser.add_argument('filelist', type=str, metavar='INPUT', help='path to input image') parser.add_argument('output_dir', type=str, metavar='OUTPUT', help='path to input image') + parser.add_argument('--co2', action='store_true', help='flag to indicate whether to run co2') args = parser.parse_args(input_args) @@ -38,23 +39,39 @@ def main(input_args=None): obs_files = [x.replace('rdn','obs') for x in rdn_files] loc_files = [x.replace('rdn','loc') for x in rdn_files] glt_files = [x.replace('rdn','glt') for x in rdn_files] + l1b_bandmask_files = [x.replace('rdn','bandmask') for x in rdn_files] + l2a_mask_files = [x.replace('l1b','l2a').replace('rdn','mask') for x in rdn_files] state_files = [x.replace('l1b','l2a').replace('rdn','statesubs') for x in rdn_files] state_files = [x if os.path.isfile(x) else None for x in state_files] - out_files = [os.path.join(args.output_dir, os.path.basename(x).split('_')[0]) for x in rdn_files] + #out_files = [os.path.join(args.output_dir, os.path.basename(x).split('_')[0]) for x in rdn_files] + for fid in files: + date = fid[4:12] + if os.path.isdir(os.path.join(args.output_dir, date)) is False: + subprocess.call(f'mkdir {os.path.join(args.output_dir, date)}',shell=True) + + out_files = [os.path.join(args.output_dir, os.path.basename(x).split('_')[0][4:12], os.path.basename(x).split('_')[0]) for x in rdn_files] for _r in range(len(rdn_files)): #for _r in range(10): - cmd_str=f'sbatch -N 1 -c 40 --mem=180G --wrap="python ghg_process.py {rdn_files[_r]} {obs_files[_r]} {loc_files[_r]} {glt_files[_r]} {out_files[_r]} --co2' - if state_files[_r] is not None: - cmd_str += f' --state_subs {state_files[_r]}"' - else: - cmd_str += f'"' + ch4_mf_kmz_file = f'{out_files[_r]}_ch4_sens_scaled_color_ort.tif' + co2_mf_kmz_file = f'{out_files[_r]}_co2_sens_scaled_color_ort.tif' + if os.path.isfile(ch4_mf_kmz_file) is False or (args.co2 and os.path.isfile(co2_mf_kmz_file) is False): + #cmd_str=f'sbatch -N 1 -c 40 --mem=180G --wrap="python ghg_process.py {rdn_files[_r]} {obs_files[_r]} {loc_files[_r]} {glt_files[_r]} {out_files[_r]} --co2' + cmd_str=f'sbatch -N 1 -c 64 -p standard --mem=300G --wrap="python ghg_process.py {rdn_files[_r]} {obs_files[_r]} {loc_files[_r]} {glt_files[_r]} {l1b_bandmask_files[_r]} {l2a_mask_files[_r]} {out_files[_r]}' - subprocess.call(cmd_str,shell=True) - time.sleep(0.1) + if args.co2: + cmd_str += ' --co2 --lut_file /store/shared/ghg/dataset_co2_full.hdf5' + + if state_files[_r] is not None: + cmd_str += f' --state_subs {state_files[_r]}"' + else: + cmd_str += f'"' + + subprocess.call(cmd_str,shell=True) + time.sleep(0.1) diff --git a/tile_all_temporal.py b/tile_all_temporal.py index 176dd9a..3f5aaa7 100644 --- a/tile_all_temporal.py +++ b/tile_all_temporal.py @@ -28,12 +28,9 @@ def main(): parser.add_argument('dates', type=str, nargs='+') parser.add_argument('--sourcedir', type=str, default='methane_20230813') parser.add_argument('--type', type=str, default='ch4', choices=['co2','ch4']) + parser.add_argument('--acquisitions_dir', type=str, default='/store/emit/ops/data/acquisitions') args = parser.parse_args() - path = os.environ['PATH'] - path = path.replace('\Library\\bin;',':') - os.environ['PATH'] = path - tiled_basedir = os.path.join(args.sourcedir, f'{args.type}_mosaic_temporal_static') sens_basedir = os.path.join(args.sourcedir, f'{args.type}_sens_mosaic_temporal_static') uncert_basedir = os.path.join(args.sourcedir, f'{args.type}_uncert_mosaic_temporal_static') @@ -43,7 +40,7 @@ def main(): if args.dates[0] == 'all': - dates = [os.path.basename(x) for x in glob.glob('/beegfs/store/emit/ops/data/acquisitions/202*')] + dates = [os.path.basename(x) for x in glob.glob(os.path.join(args.acquisitions_dir,'202*'))] else: dates = args.dates @@ -65,9 +62,9 @@ def main(): od_date = f'{date[:4]}-{date[4:6]}-{date[6:8]}T00_00_01Z-to-{date[:4]}-{date[4:6]}-{date[6:8]}T23_59_59Z' - subprocess.call(f'sbatch -N 1 -c 40 --mem=180G --job-name {args.type}_tile_{date} --wrap="python daily_tiler.py {static_file_list} {os.path.join(tiled_basedir, od_date)}"',shell=True) - subprocess.call(f'sbatch -N 1 -c 40 --mem=180G --job-name {args.type}_tile_{date} --wrap="python daily_tiler.py {sens_file_list} {os.path.join(sens_basedir, od_date)}"',shell=True) - subprocess.call(f'sbatch -N 1 -c 40 --mem=180G --job-name {args.type}_tile_{date} --wrap="python daily_tiler.py {uncert_file_list} {os.path.join(uncert_basedir, od_date)}"',shell=True) + subprocess.call(f'sbatch -N 1 -c 64 --mem=370G --job-name {args.type}_tile_{date} --wrap="python daily_tiler.py {static_file_list} {os.path.join(tiled_basedir, od_date)}"',shell=True) + subprocess.call(f'sbatch -N 1 -c 64 --mem=370G --job-name {args.type}_tile_{date} --wrap="python daily_tiler.py {sens_file_list} {os.path.join(sens_basedir, od_date)}"',shell=True) + subprocess.call(f'sbatch -N 1 -c 64 --mem=370G --job-name {args.type}_tile_{date} --wrap="python daily_tiler.py {uncert_file_list} {os.path.join(uncert_basedir, od_date)}"',shell=True)