updates for migration ops

daily_tiler.py

@@ -31,7 +31,6 @@ def main(rawargs=None):
     parser = argparse.ArgumentParser(description="Daily Tiler")
     parser.add_argument('file_list',type=str)
     parser.add_argument('output_dest', type=str)
-    parser.add_argument('--dynamic', action='store_true')
     args = parser.parse_args(rawargs)
 
     files = open(args.file_list,'r').readlines()
@@ -42,10 +41,7 @@ def main(rawargs=None):
         count = 0
         success=False
 
-        if args.dynamic:
-            subprocess.call(f'python /beegfs/scratch/brodrick/emit/MMGIS/auxiliary/gdal2customtiles/gdal2tiles_3.5.2.py --dem {fi} {args.output_dest} -r near-composite -z 5-11 --srcnodata=-9999 --processes=40 && echo {fi} >> {os.path.splitext(args.file_list)[0] + "_completed.txt"}',shell=True)
-        else:
-            subprocess.call(f'gdal2tiles.py {fi} {args.output_dest} -z 4-12 --srcnodata 0 --processes=40 -r antialias -x && echo {fi} >> {os.path.splitext(args.file_list)[0] + "_completed.txt"}',shell=True)
+        subprocess.call(f'gdal2tiles.py {fi} {args.output_dest} -z 4-12 --srcnodata 0 --processes=40 -r antialias -x && echo {fi} >> {os.path.splitext(args.file_list)[0] + "_completed.txt"}',shell=True)
 
 
 

delivery_plume_tiler.py

@@ -186,13 +186,14 @@ def main(input_args=None):
     parser = argparse.ArgumentParser(description="Delineate/colorize plume")
     parser.add_argument('--source_dir', type=str, default='methane_20230813')
     parser.add_argument('--dest_dir', type=str, default='visions_delivery')
-    parser.add_argument('--manual_del_dir', type=str, default='/beegfs/scratch/brodrick/methane/ch4_plumedir_scenetest/')
+    parser.add_argument('--manual_del_dir', type=str, default='/store/brodrick/methane/ch4_plumedir_scenetest/')
     parser.add_argument('--software_version', type=str, default=None)
     parser.add_argument('--data_version', type=str, default=None)
     parser.add_argument('--visions_delivery', type=int, choices=[0,1,2],default=0)
     parser.add_argument('--n_cores', type=int, default=1)
     parser.add_argument('--overwrite', action='store_true')
-    parser.add_argument('--previous_plume_file', type=str, default='visions_delivery/plume_list.txt')
+    parser.add_argument('--delivery_dir', type=str, default='visions_delivery')
+    parser.add_argument('--previous_plume_file', type=str, default='plume_list.txt')
     parser.add_argument('--loglevel', type=str, default='DEBUG', help='logging verbosity')    
     parser.add_argument('--logfile', type=str, default=None, help='output file to write log to')    
     args = parser.parse_args(input_args)
@@ -202,7 +203,7 @@ def main(input_args=None):
 
     tile_dir = os.path.join(args.dest_dir, 'ch4_plume_tiles')
 
-    with open(args.previous_plume_file,'r') as f:
+    with open(os.path.join(args.delivery_dir, args.previous_plume_file),'r') as f:
         delivered_plume_names = f.read().splitlines()
         delivered_plume_ids = [f'CH4_PlumeComplex-{x.split("_")[-1]}' for x in delivered_plume_names]
 
@@ -318,7 +319,7 @@ def main(input_args=None):
 
         with open(os.path.join(args.dest_dir, 'combined_plume_metadata.json'), 'w') as fout:
             fout.write(json.dumps(outdict, cls=SerialEncoder)) 
-        subprocess.call("rsync visions_delivery/combined_plume_metadata.json brodrick@${EMIT_SCIENCE_IP}:/data/emit/mmgis/coverage/combined_plume_metadata.json",shell=True)
+        subprocess.call(f"rsync {args.delivery_dir}/combined_plume_metadata.json brodrick@${EMIT_SCIENCE_IP}:/data/emit/mmgis/coverage/combined_plume_metadata.json",shell=True)
 
 
         logging.info('Tile output')
@@ -329,7 +330,7 @@ def main(input_args=None):
             subfeatures = [feat for _feat, feat in enumerate(all_plume_meta['features']) if _feat in match_idx and _feat in valid_plume_idx]
             if len(subfeatures) > 0:
                 tile_dcid(subfeatures, outdir, args.manual_del_dir)
-        subprocess.call("rsync -a --info=progress2 visions_delivery/visions_ch4_tiles/ brodrick@${EMIT_SCIENCE_IP}:/data/emit/mmgis/mosaics/ch4_plume_tiles/",shell=True)
+        subprocess.call(f"rsync -a --info=progress2 {outdir} brodrick@${EMIT_SCIENCE_IP}:/data/emit/mmgis/mosaics/ch4_plume_tiles/",shell=True)
 
 
 

run_bulk_ghg.py

@@ -58,10 +58,10 @@ def main(input_args=None):
 
       launch = os.path.isfile(ch4_mf_kmz_file) is False
       if os.path.isfile(ch4_mf_kmz_file) is False or (args.co2 and os.path.isfile(co2_mf_kmz_file) is False):
-        cmd_str=f'sbatch -N 1 -c 40 -p standard --mem=180G --wrap="source /tmp/miniconda/bin/activate; conda activate isofit_env; python ghg_process.py {rdn_files[_r]} {obs_files[_r]} {loc_files[_r]} {glt_files[_r]} {l1b_bandmask_files[_r]} {l2a_mask_files[_r]} {out_files[_r]}'
+        cmd_str=f'sbatch -N 1 -c 64 -p standard --mem=300G --wrap="python ghg_process.py {rdn_files[_r]} {obs_files[_r]} {loc_files[_r]} {glt_files[_r]} {l1b_bandmask_files[_r]} {l2a_mask_files[_r]} {out_files[_r]}'
 
         if args.co2:
-            cmd_str += ' --co2'
+            cmd_str += ' --co2 --lut_file /store/shared/ghg/dataset_co2_full.hdf5'
 
         if state_files[_r] is not None:
             cmd_str += f' --state_subs {state_files[_r]}"'

run_plumeid_ghg.py

@@ -36,12 +36,12 @@ def main(input_args=None):
 
     plumedict = json.load(open(args.methane_metadata,'r'))
 
-    fids = [x['properties']['Scene FID'] for x in plumedict['features']]
+    #fids = [x['properties']['Scene FIDs'] for x in plumedict['features']]
+    fids = np.unique([sublist for feat in plumedict['features'] for sublist in feat['properties']['Scene FIDs']])
 
-    un_fids = np.unique(fids)
     rdn_files = []
-    for fid in un_fids:
-        glist = glob(f'/beegfs/store/emit/ops/data/acquisitions/{fid[4:12]}/{fid}/l1b/*_rdn_b0106_v01.img')
+    for fid in fids:
+        glist = glob(f'/store/emit/ops/data/acquisitions/{fid[4:12]}/{fid}/l1b/*_rdn_b0106_v01.img')
         if len(glist) > 0:
             rdn_files.append(glist[0])
         else:
@@ -50,7 +50,7 @@ def main(input_args=None):
         if rdn_files[_fid] is None:
             rdn_files.pop(_fid)
             fids.pop(_fid)
-    un_fids = np.unique(fids)
+    fids = np.unique(fids)
 
     obs_files = [x.replace('rdn','obs') for x in rdn_files]
     loc_files = [x.replace('rdn','loc') for x in rdn_files]
@@ -61,7 +61,7 @@ def main(input_args=None):
     state_files = [x.replace('l1b','l2a').replace('rdn','statesubs') for x in rdn_files]
     state_files = [x if os.path.isfile(x) else None for x in state_files]
 
-    for fid in un_fids:
+    for fid in fids:
         date = fid[4:12]
         if os.path.isdir(os.path.join(args.output_dir, date)) is False:
             subprocess.call(f'mkdir {os.path.join(args.output_dir, date)}',shell=True)
@@ -76,10 +76,10 @@ def main(input_args=None):
 
       launch = os.path.isfile(ch4_mf_kmz_file) is False
       if os.path.isfile(ch4_mf_kmz_file) is False or (args.co2 and os.path.isfile(co2_mf_kmz_file) is False):
-        cmd_str=f'sbatch -N 1 -c 40 -p standard --mem=180G --wrap="python ghg_process.py {rdn_files[_r]} {obs_files[_r]} {loc_files[_r]} {glt_files[_r]} {l1b_bandmask_files[_r]} {l2a_mask_files[_r]} {out_files[_r]}'
+        cmd_str=f'sbatch -N 1 -c 64 -p standard --mem=300G --wrap="python ghg_process.py {rdn_files[_r]} {obs_files[_r]} {loc_files[_r]} {glt_files[_r]} {l1b_bandmask_files[_r]} {l2a_mask_files[_r]} {out_files[_r]}'
 
         if args.co2:
-            cmd_str += ' --co2'
+            cmd_str += ' --co2 --lut_file /store/shared/ghg/dataset_co2_full.hdf5'
 
         if state_files[_r] is not None:
             cmd_str += f' --state_subs {state_files[_r]}"'

run_priority.py

@@ -30,6 +30,7 @@ def main(input_args=None):
     parser = argparse.ArgumentParser(description="Robust MF")
     parser.add_argument('filelist', type=str,  metavar='INPUT', help='path to input image')   
     parser.add_argument('output_dir', type=str,  metavar='OUTPUT', help='path to input image')   
+    parser.add_argument('--co2', action='store_true', help='flag to indicate whether to run co2')    
     args = parser.parse_args(input_args)
 
 
@@ -38,23 +39,39 @@ def main(input_args=None):
     obs_files = [x.replace('rdn','obs') for x in rdn_files]
     loc_files = [x.replace('rdn','loc') for x in rdn_files]
     glt_files = [x.replace('rdn','glt') for x in rdn_files]
+    l1b_bandmask_files = [x.replace('rdn','bandmask') for x in rdn_files]
+    l2a_mask_files = [x.replace('l1b','l2a').replace('rdn','mask') for x in rdn_files]
 
     state_files = [x.replace('l1b','l2a').replace('rdn','statesubs') for x in rdn_files]
     state_files = [x if os.path.isfile(x) else None for x in state_files]
 
-    out_files = [os.path.join(args.output_dir, os.path.basename(x).split('_')[0]) for x in rdn_files]
+    #out_files = [os.path.join(args.output_dir, os.path.basename(x).split('_')[0]) for x in rdn_files]
+    for fid in files:
+        date = fid[4:12]
+        if os.path.isdir(os.path.join(args.output_dir, date)) is False:
+            subprocess.call(f'mkdir {os.path.join(args.output_dir, date)}',shell=True)
+
+    out_files = [os.path.join(args.output_dir, os.path.basename(x).split('_')[0][4:12], os.path.basename(x).split('_')[0]) for x in rdn_files]
 
     for _r in range(len(rdn_files)):
     #for _r in range(10):
 
-        cmd_str=f'sbatch -N 1 -c 40 --mem=180G --wrap="python ghg_process.py {rdn_files[_r]} {obs_files[_r]} {loc_files[_r]} {glt_files[_r]} {out_files[_r]} --co2'
-        if state_files[_r] is not None:
-            cmd_str += f' --state_subs {state_files[_r]}"'
-        else:
-            cmd_str += f'"'
+        ch4_mf_kmz_file = f'{out_files[_r]}_ch4_sens_scaled_color_ort.tif'
+        co2_mf_kmz_file = f'{out_files[_r]}_co2_sens_scaled_color_ort.tif'
+        if os.path.isfile(ch4_mf_kmz_file) is False or (args.co2 and os.path.isfile(co2_mf_kmz_file) is False):
+            #cmd_str=f'sbatch -N 1 -c 40 --mem=180G --wrap="python ghg_process.py {rdn_files[_r]} {obs_files[_r]} {loc_files[_r]} {glt_files[_r]} {out_files[_r]} --co2'
+            cmd_str=f'sbatch -N 1 -c 64 -p standard --mem=300G --wrap="python ghg_process.py {rdn_files[_r]} {obs_files[_r]} {loc_files[_r]} {glt_files[_r]} {l1b_bandmask_files[_r]} {l2a_mask_files[_r]} {out_files[_r]}'
 
-        subprocess.call(cmd_str,shell=True)
-        time.sleep(0.1)
+            if args.co2:
+                cmd_str += ' --co2 --lut_file /store/shared/ghg/dataset_co2_full.hdf5'
+
+            if state_files[_r] is not None:
+                cmd_str += f' --state_subs {state_files[_r]}"'
+            else:
+                cmd_str += f'"'
+
+            subprocess.call(cmd_str,shell=True)
+            time.sleep(0.1)
 
 
 

tile_all_temporal.py

@@ -28,12 +28,9 @@ def main():
     parser.add_argument('dates', type=str, nargs='+')
     parser.add_argument('--sourcedir', type=str, default='methane_20230813')
     parser.add_argument('--type', type=str, default='ch4', choices=['co2','ch4'])
+    parser.add_argument('--acquisitions_dir', type=str, default='/store/emit/ops/data/acquisitions')
     args = parser.parse_args()
 
-    path = os.environ['PATH']
-    path = path.replace('\Library\\bin;',':')
-    os.environ['PATH'] = path
-
     tiled_basedir = os.path.join(args.sourcedir, f'{args.type}_mosaic_temporal_static')
     sens_basedir = os.path.join(args.sourcedir, f'{args.type}_sens_mosaic_temporal_static')
     uncert_basedir = os.path.join(args.sourcedir, f'{args.type}_uncert_mosaic_temporal_static')
@@ -43,7 +40,7 @@ def main():
 
 
     if args.dates[0] == 'all':
-        dates = [os.path.basename(x) for x in glob.glob('/beegfs/store/emit/ops/data/acquisitions/202*')]
+        dates = [os.path.basename(x) for x in glob.glob(os.path.join(args.acquisitions_dir,'202*'))]
     else:
         dates = args.dates
 
@@ -65,9 +62,9 @@ def main():
         od_date = f'{date[:4]}-{date[4:6]}-{date[6:8]}T00_00_01Z-to-{date[:4]}-{date[4:6]}-{date[6:8]}T23_59_59Z'
 
 
-        subprocess.call(f'sbatch -N 1 -c 40 --mem=180G --job-name {args.type}_tile_{date} --wrap="python daily_tiler.py {static_file_list} {os.path.join(tiled_basedir, od_date)}"',shell=True)
-        subprocess.call(f'sbatch -N 1 -c 40 --mem=180G --job-name {args.type}_tile_{date} --wrap="python daily_tiler.py {sens_file_list}   {os.path.join(sens_basedir, od_date)}"',shell=True)
-        subprocess.call(f'sbatch -N 1 -c 40 --mem=180G --job-name {args.type}_tile_{date} --wrap="python daily_tiler.py {uncert_file_list} {os.path.join(uncert_basedir, od_date)}"',shell=True)
+        subprocess.call(f'sbatch -N 1 -c 64 --mem=370G --job-name {args.type}_tile_{date} --wrap="python daily_tiler.py {static_file_list} {os.path.join(tiled_basedir, od_date)}"',shell=True)
+        subprocess.call(f'sbatch -N 1 -c 64 --mem=370G --job-name {args.type}_tile_{date} --wrap="python daily_tiler.py {sens_file_list}   {os.path.join(sens_basedir, od_date)}"',shell=True)
+        subprocess.call(f'sbatch -N 1 -c 64 --mem=370G --job-name {args.type}_tile_{date} --wrap="python daily_tiler.py {uncert_file_list} {os.path.join(uncert_basedir, od_date)}"',shell=True)