diff --git a/recipes/deploy/deploy_cloud_2025.yaml b/recipes/deploy/deploy_cloud_2025.yaml deleted file mode 100644 index 382c0e2..0000000 --- a/recipes/deploy/deploy_cloud_2025.yaml +++ /dev/null @@ -1,32 +0,0 @@ -flow_start_time: null - -source: - mode: git - git: - url: https://github.com/OSOceanAcoustics/echodataflow.git - branch: rewrite_2026_cleanup - -default_work_pool_name: local -flows: - ingest_haul: - module: flows_biology - deployment_name: ingest_haul - interval: 5 - ingest_NASC: - module: flows_integration - deployment_name: ingest_NASC - interval: 5 - update_grid: - module: flows_integration - deployment_name: update_grid - triggers: - - expect: haul.ingested - resource_name: ingest_haul - - expect: nasc.ingested - resource_name: ingest_NASC - update_cache_MVBS: - module: flows_viz_cloud - deployment_name: update_cache_MVBS - interval: 10 - cron_offset: 3 - inject_time_offset: true diff --git a/recipes/deploy/deploy_cloud_drixo16_2026.yaml b/recipes/deploy/deploy_cloud_drixo16_2026.yaml deleted file mode 100644 index 5211394..0000000 --- a/recipes/deploy/deploy_cloud_drixo16_2026.yaml +++ /dev/null @@ -1,16 +0,0 @@ -flow_start_time: "2026-02-09T02:00:00+00:00" - -source: - mode: git - git: - url: https://github.com/echostack-org/echodataflow.git - branch: drixo16_2026 - -default_work_pool_name: local -flows: - update_cache_MVBS: - module: flows_viz_cloud - deployment_name: update_cache_MVBS - interval: 10 - cron_offset: 3 - inject_time_offset: true diff --git a/recipes/deploy/deploy_cloud_iwcps_2026.yaml b/recipes/deploy/deploy_cloud_iwcps_2026.yaml deleted file mode 100644 index 7f8c018..0000000 --- a/recipes/deploy/deploy_cloud_iwcps_2026.yaml +++ /dev/null @@ -1,32 +0,0 @@ -flow_start_time: null - -source: - mode: git - git: - url: https://github.com/OSOceanAcoustics/echodataflow.git - branch: main - -default_work_pool_name: local -flows: - # ingest_haul: - # module: flows_biology - # deployment_name: ingest_haul - # interval: 5 - # ingest_NASC: - # module: flows_integration - # deployment_name: ingest_NASC - # interval: 5 - # update_grid: - # module: flows_integration - # deployment_name: update_grid - # triggers: - # - expect: haul.ingested - # resource_name: ingest_haul - # - expect: nasc.ingested - # resource_name: ingest_NASC - update_cache_MVBS: - module: flows_viz_cloud - deployment_name: update_cache_MVBS - interval: 10 - cron_offset: 3 - inject_time_offset: true diff --git a/recipes/deploy/deploy_cloud_iwcps_dyson_2026.yaml b/recipes/deploy/deploy_cloud_iwcps_dyson_2026.yaml deleted file mode 100644 index e87e9de..0000000 --- a/recipes/deploy/deploy_cloud_iwcps_dyson_2026.yaml +++ /dev/null @@ -1,40 +0,0 @@ -flow_start_time: null - -source: - mode: git - git: - url: https://github.com/OSOceanAcoustics/echodataflow.git - branch: main - -default_work_pool_name: local -flows: - # ingest_haul: - # module: flows_biology - # deployment_name: ingest_haul - # interval: 5 - # ingest_NASC: - # module: flows_integration - # deployment_name: ingest_NASC - # interval: 5 - # update_grid: - # module: flows_integration - # deployment_name: update_grid - # triggers: - # - expect: haul.ingested - # resource_name: ingest_haul - # - expect: nasc.ingested - # resource_name: ingest_NASC - update_cache_MVBS_lasker: - module: flows_viz_cloud - flow_alias: update_cache_MVBS - deployment_name: update_cache_MVBS_lasker - interval: 10 - cron_offset: 3 - inject_time_offset: true - update_cache_MVBS_dyson: - module: flows_viz_cloud - flow_alias: update_cache_MVBS - deployment_name: update_cache_MVBS_dyson - interval: 10 - cron_offset: 3 - inject_time_offset: true diff --git a/recipes/deploy/deploy_edge_dyson_2026.yaml b/recipes/deploy/deploy_edge_dyson_2026.yaml deleted file mode 100644 index 1af0f64..0000000 --- a/recipes/deploy/deploy_edge_dyson_2026.yaml +++ /dev/null @@ -1,31 +0,0 @@ -flow_start_time: - -source: - mode: git - git: - url: https://github.com/echostack-org/echodataflow.git - branch: origin/main - -default_work_pool_name: local -flows: - # copy_raw: - # module: flows_helper - # deployment_name: copy_raw - # interval: 2 - raw2Sv: - module: flows_acoustics - deployment_name: raw2Sv - interval: 5 - create_MVBS: - module: flows_acoustics - deployment_name: create-MVBS - interval: 10 - cron_offset: 3 - inject_time_offset: true - file_upload_acoustics: - module: flows_helper - deployment_name: file-upload-acoustics - flow_alias: file_upload - interval: 10 - apply_separately: true - work_pool_name: local diff --git a/recipes/deploy/deploy_edge_iwcps_2025.yaml b/recipes/deploy/deploy_edge_iwcps_2025.yaml deleted file mode 100644 index f9c9db0..0000000 --- a/recipes/deploy/deploy_edge_iwcps_2025.yaml +++ /dev/null @@ -1,44 +0,0 @@ -flow_start_time: "2025-07-04T16:13:00+00:00" - -source: - mode: git - git: - url: https://github.com/OSOceanAcoustics/echodataflow.git - branch: rewrite_2026_cleanup - -default_work_pool_name: local -flows: - copy_raw: - module: flows_helper - deployment_name: copy_raw - interval: 2 - raw2Sv: - module: flows_acoustics - deployment_name: raw2Sv_leg2 - interval: 5 - create_MVBS: - module: flows_acoustics - deployment_name: create-MVBS_leg2 - interval: 10 - cron_offset: 3 - inject_time_offset: true - predict_hake: - module: flows_acoustics - deployment_name: predict-hake_leg2 - interval: 20 - inject_time_offset: true - file_upload_acoustics: - module: flows_helper - deployment_name: file-upload-acoustics_leg2 - flow_alias: file_upload - interval: 10 - apply_separately: true - work_pool_name: local - file_upload_trawl: - module: flows_helper - deployment_name: file-upload-trawl_20250902 - flow_alias: file_upload - interval: 10 - cron_offset: 3 - apply_separately: true - work_pool_name: local diff --git a/recipes/deploy/deploy_edge_iwcps_2026.yaml b/recipes/deploy/deploy_edge_iwcps_2026.yaml deleted file mode 100644 index e3f98f1..0000000 --- a/recipes/deploy/deploy_edge_iwcps_2026.yaml +++ /dev/null @@ -1,45 +0,0 @@ -flow_start_time: #"2026-07-01T00:00:00+00:00" # time travel for testing or reprocessing - # use ISO8601 format: if UTC use +00:00 or Z - -source: - mode: git - git: - url: https://github.com/echostack-org/echodataflow.git - branch: origin/main - -default_work_pool_name: local -flows: - # copy_raw: - # module: flows_helper - # deployment_name: copy_raw - # interval: 2 - raw2Sv: - module: flows_acoustics - deployment_name: raw2Sv - interval: 5 - create_MVBS: - module: flows_acoustics - deployment_name: create-MVBS - interval: 10 - cron_offset: 3 - inject_time_offset: true - predict_hake: - module: flows_acoustics - deployment_name: predict-hake - interval: 20 - inject_time_offset: true - file_upload_acoustics: - module: flows_helper - deployment_name: file-upload-acoustics - flow_alias: file_upload - interval: 10 - apply_separately: true - work_pool_name: local - # file_upload_trawl: - # module: flows_helper - # deployment_name: file-upload-trawl_20250902 - # flow_alias: file_upload - # interval: 10 - # cron_offset: 3 - # apply_separately: true - # work_pool_name: local \ No newline at end of file diff --git a/recipes/params/params_cloud_2025.yaml b/recipes/params/params_cloud_2025.yaml deleted file mode 100644 index 9b70fce..0000000 --- a/recipes/params/params_cloud_2025.yaml +++ /dev/null @@ -1,33 +0,0 @@ -init: - counter_raw_copy: 0 - -flows: - ingest_haul: - path_vm_local: "/media/volume/shimada_202506_volume/integration" # path to local integrated data - path_bio_files: "agr230002-bucket01/prefect_sh2506_test/trawl" # path to cloud trawl data - cred_file: "/home/exouser/.config/rclone/rclone.conf" - file_haul_info_all: haul_info_all.csv - file_specimen_all: specimen_all.csv - file_length_all: length_all.csv - file_length_count_all: length_count_all.csv - file_stratum_mean: stratum_mean.csv - ingest_NASC: - path_vm_local: "/media/volume/shimada_202506_volume/integration" # path to local integrated data - path_NASC_files: "agr230002-bucket01/prefect_sh2506_test/acoustics/NASC" # path to cloud acoustic data - cred_file: "/home/exouser/.config/rclone/rclone.conf" - file_NASC_all: NASC_all.csv - file_stratum_mean: stratum_mean.csv - file_NASC_all_grid: NASC_all_griddify.geojson - num_NASC_reprocess: 1 - new_file_num_limit: -1 - update_grid: - path_vm_local: "/media/volume/shimada_202506_volume/integration" # path to local integrated data - file_NASC_all_grid: NASC_all_griddify.geojson - file_stratum_mean: stratum_mean.csv - update_cache_MVBS: - slice_mins: 180 - path_cache: "/media/volume/shimada_202506_volume/viz_data_cache" - path_MVBS: "agr230002-bucket01/prefect_sh2506_test/acoustics/MVBS" - cred_file: "/home/exouser/.config/rclone/rclone.conf" - file_MVBS_csv: MVBS_files.csv - file_MVBS_zarr: latest_MVBS.zarr diff --git a/recipes/params/params_cloud_dirxo16_2026.yaml b/recipes/params/params_cloud_dirxo16_2026.yaml deleted file mode 100644 index fec43fa..0000000 --- a/recipes/params/params_cloud_dirxo16_2026.yaml +++ /dev/null @@ -1,11 +0,0 @@ -init: - counter_raw_copy: 0 - -flows: - update_cache_MVBS: - slice_mins: 180 - path_cache: "/media/volume/shimada_202506_volume/viz_data_cache_2026/drixo16" - path_MVBS: "agr230002-bucket01/prefect_drixo16_2026/acoustics/MVBS" - cred_file: "/home/exouser/.config/rclone/rclone.conf" - file_MVBS_csv: MVBS_files.csv - file_MVBS_zarr: latest_MVBS.zarr diff --git a/recipes/params/params_cloud_iwcps_2026.yaml b/recipes/params/params_cloud_iwcps_2026.yaml deleted file mode 100644 index bbec7a7..0000000 --- a/recipes/params/params_cloud_iwcps_2026.yaml +++ /dev/null @@ -1,33 +0,0 @@ -init: - counter_raw_copy: 0 - -flows: - # ingest_haul: - # path_vm_local: "/media/volume/shimada_202506_volume/integration" # path to local integrated data - # path_bio_files: "agr230002-bucket01/prefect_sh2506_test/trawl" # path to cloud trawl data - # cred_file: "/home/exouser/.config/rclone/rclone.conf" - # file_haul_info_all: haul_info_all.csv - # file_specimen_all: specimen_all.csv - # file_length_all: length_all.csv - # file_length_count_all: length_count_all.csv - # file_stratum_mean: stratum_mean.csv - # ingest_NASC: - # path_vm_local: "/media/volume/shimada_202506_volume/integration" # path to local integrated data - # path_NASC_files: "agr230002-bucket01/prefect_sh2506_test/acoustics/NASC" # path to cloud acoustic data - # cred_file: "/home/exouser/.config/rclone/rclone.conf" - # file_NASC_all: NASC_all.csv - # file_stratum_mean: stratum_mean.csv - # file_NASC_all_grid: NASC_all_griddify.geojson - # num_NASC_reprocess: 1 - # new_file_num_limit: -1 - # update_grid: - # path_vm_local: "/media/volume/shimada_202506_volume/integration" # path to local integrated data - # file_NASC_all_grid: NASC_all_griddify.geojson - # file_stratum_mean: stratum_mean.csv - update_cache_MVBS: - slice_mins: 180 - path_cache: "/media/volume/shimada_202506_volume/viz_data_cache_2026/iwcsp_2026" - path_MVBS: "agr230002-bucket01/prefect_IWCPS_2026_new/acoustics/MVBS" - cred_file: "/home/exouser/.config/rclone/rclone.conf" - file_MVBS_csv: MVBS_files.csv - file_MVBS_zarr: latest_MVBS.zarr diff --git a/recipes/params/params_cloud_iwcps_dyson_2026.yaml b/recipes/params/params_cloud_iwcps_dyson_2026.yaml deleted file mode 100644 index a116dfb..0000000 --- a/recipes/params/params_cloud_iwcps_dyson_2026.yaml +++ /dev/null @@ -1,40 +0,0 @@ -init: - counter_raw_copy: 0 - -flows: - # ingest_haul: - # path_vm_local: "/media/volume/shimada_202506_volume/integration" # path to local integrated data - # path_bio_files: "agr230002-bucket01/prefect_sh2506_test/trawl" # path to cloud trawl data - # cred_file: "/home/exouser/.config/rclone/rclone.conf" - # file_haul_info_all: haul_info_all.csv - # file_specimen_all: specimen_all.csv - # file_length_all: length_all.csv - # file_length_count_all: length_count_all.csv - # file_stratum_mean: stratum_mean.csv - # ingest_NASC: - # path_vm_local: "/media/volume/shimada_202506_volume/integration" # path to local integrated data - # path_NASC_files: "agr230002-bucket01/prefect_sh2506_test/acoustics/NASC" # path to cloud acoustic data - # cred_file: "/home/exouser/.config/rclone/rclone.conf" - # file_NASC_all: NASC_all.csv - # file_stratum_mean: stratum_mean.csv - # file_NASC_all_grid: NASC_all_griddify.geojson - # num_NASC_reprocess: 1 - # new_file_num_limit: -1 - # update_grid: - # path_vm_local: "/media/volume/shimada_202506_volume/integration" # path to local integrated data - # file_NASC_all_grid: NASC_all_griddify.geojson - # file_stratum_mean: stratum_mean.csv - update_cache_MVBS_lasker: - slice_mins: 180 - path_cache: "/media/volume/shimada_202506_volume/viz_data_cache_2026/iwcsp_2026" - path_MVBS: "agr230002-bucket01/prefect_IWCPS_2026_new/acoustics/MVBS" - cred_file: "/home/exouser/.config/rclone/rclone.conf" - file_MVBS_csv: MVBS_files.csv - file_MVBS_zarr: latest_MVBS.zarr - update_cache_MVBS_dyson: - slice_mins: 180 - path_cache: "/media/volume/shimada_202506_volume/viz_data_cache_2026/dyson_2026" - path_MVBS: "agr230002-bucket01/prefect_MACE_2026/acoustics/MVBS" - cred_file: "/home/exouser/.config/rclone/rclone.conf" - file_MVBS_csv: MVBS_files.csv - file_MVBS_zarr: latest_MVBS.zarr diff --git a/recipes/params/params_edge_dyson_2026.yaml b/recipes/params/params_edge_dyson_2026.yaml deleted file mode 100644 index 6361ee0..0000000 --- a/recipes/params/params_edge_dyson_2026.yaml +++ /dev/null @@ -1,34 +0,0 @@ -init: # TODO: consider moving this section to deploy_ship.yaml - counter_raw_copy: 0 - -flows: - raw2Sv: - exclude_before: "2026-06-28T00:00:00+00:00" # use ISO8601 format: if UTC use +00:00 or Z - exclude_raw_file: [] - # - "IWCPS-D20250617-T175659.raw" - # - "IWCPS-D20250711-T212420.raw" - # - "IWCPS-D20250901-T145533.raw" - parallel: False - encode_mode: power - waveform_mode: CW - depth_offset: 7 - sonar_model: EK80 - nmea_sentence: GGA - filename_pattern: "DY2604-*.raw" # .raw filename pattern to include - path_raw: "/Volumes/DY2604/reprocessed-cw" # path to raw files - path_main: "/Users/feresa/mace_workflow_output" - file_Sv_csv: Sv_files.csv - create_MVBS: - slice_mins: 20 - num_slices: 3 - range_bin: 0.25m - ping_time_bin: 5s - path_main: "/Users/feresa/mace_workflow_output" - file_Sv_csv: Sv_files.csv - file_MVBS_csv: MVBS_files.csv - file_upload_acoustics: - src_dir: "/Users/feresa/mace_workflow_output" - dest_dir: "osn_sdsc_hake:/agr230002-bucket01/prefect_MACE_2026/acoustics" - exclude_subdirs: - - Sv - max_age: 2 diff --git a/recipes/params/params_edge_iwcps_2025.yaml b/recipes/params/params_edge_iwcps_2025.yaml deleted file mode 100644 index d28a284..0000000 --- a/recipes/params/params_edge_iwcps_2025.yaml +++ /dev/null @@ -1,48 +0,0 @@ -init: # TODO: consider moving this section to deploy_ship.yaml - counter_raw_copy: 0 - -flows: - raw2Sv: - exclude_before: "2025-06-17T20:00:00+00:00" # "2025-06-13T10:00:00+00:00" # use ISO8601 format: if UTC use +00:00 or Z - exclude_raw_file: - - "IWCPS-D20250617-T175659.raw" - - "IWCPS-D20250711-T212420.raw" - - "IWCPS-D20250901-T145533.raw" - parallel: False - encode_mode: power - waveform_mode: CW - depth_offset: 7 - sonar_model: EK80 - nmea_sentence: GGA - filename_pattern: "IWCPS*.raw" - path_raw: "/Users/feresa/mnt_qnap/Leg 2" - path_main: "/Volumes/sh2506_ext1/leg2" - file_Sv_csv: Sv_files.csv - create_MVBS: - slice_mins: 20 - num_slices: 3 - path_main: "/Volumes/sh2506_ext1/leg2" - file_Sv_csv: Sv_files.csv - file_MVBS_csv: MVBS_files.csv - predict_hake: - slice_mins: 40 - num_slices: 3 - temperature: 0.5 - softmax_threshold: 0.5 - path_main: "/Volumes/sh2506_ext1/leg2" - file_MVBS_csv: MVBS_files.csv - file_prediction_csv: prediction_files.csv - file_upload_acoustics: - src_dir: "/Volumes/sh2506_ext1/leg2" - dest_dir: "osn_sdsc_hake:/agr230002-bucket01/prefect_sh2506_test/acoustics" - exclude_subdirs: - - Sv - max_age: 2 - file_upload_trawl: - src_dir: "/Users/feresa/mnt_qnap_clams/nwc" - dest_dir: "osn_sdsc_hake:/agr230002-bucket01/prefect_sh2506_test/trawl" - exclude_subdirs: - - backups - - "temp AB stuff" - - "How to run CLAMS reports.docx" - max_age: -1 diff --git a/recipes/params/params_edge_iwcps_2026.yaml b/recipes/params/params_edge_iwcps_2026.yaml deleted file mode 100644 index 565e1ca..0000000 --- a/recipes/params/params_edge_iwcps_2026.yaml +++ /dev/null @@ -1,48 +0,0 @@ -flows: - raw2Sv: - exclude_before: "2026-07-01T00:00:00+00:00" # use ISO8601 format: if UTC use +00:00 or Z - exclude_raw_file: # any known corrupted files, use [] if none - - "RL2604-D20260726-T233600.raw" - parallel: False - encode_mode: power - waveform_mode: CW - depth_offset: 7 - sonar_model: EK80 - datagram_type: MRU1 - filename_pattern: "RL2604-*.raw" # .raw filename pattern to include - path_raw: "/Users/phocoena/data_import/Cruise Data/RL2604 IWCPS/RAW/EK80" # path to raw files - path_main: /Volumes/sh2506_ext1/edge2cloud_workflow/2026_IWCPS_output - file_Sv_csv: Sv_files.csv - create_MVBS: - slice_mins: 20 - num_slices: 3 - range_bin: 1m - ping_time_bin: 5s - path_main: /Volumes/sh2506_ext1/edge2cloud_workflow/2026_IWCPS_output - file_Sv_csv: Sv_files.csv - file_MVBS_csv: MVBS_files.csv - predict_hake: - slice_mins: 40 - num_slices: 3 - temperature: 0.5 - softmax_threshold: 0.5 - max_depth: 590.0 - path_weight: /Volumes/sh2506_ext1/edge2cloud_workflow/model_weights/binary_hake_model_weight.ckpt - path_main: /Volumes/sh2506_ext1/edge2cloud_workflow/2026_IWCPS_output - file_MVBS_csv: MVBS_files.csv - file_prediction_csv: prediction_files.csv - file_upload_acoustics: - src_dir: /Volumes/sh2506_ext1/edge2cloud_workflow/2026_IWCPS_output - dest_dir: "osn_sdsc_hake:/agr230002-bucket01/prefect_IWCPS_2026_new/acoustics" - exclude_subdirs: - - Sv - - model_weights - max_age: 2 - # file_upload_trawl: - # src_dir: "/Users/feresa/mnt_qnap_clams/nwc" - # dest_dir: "osn_sdsc_hake:/agr230002-bucket01/prefect_sh2506_test/trawl" - # exclude_subdirs: - # - backups - # - "temp AB stuff" - # - "How to run CLAMS reports.docx" - # max_age: -1 \ No newline at end of file diff --git a/recipes/resources/inpfc_def.csv b/recipes/resources/inpfc_def.csv deleted file mode 100644 index 15ddac1..0000000 --- a/recipes/resources/inpfc_def.csv +++ /dev/null @@ -1,7 +0,0 @@ -stratum_num,latitude_northern_limit,stratum_name -1,36,Conception -2,40.5,Monterey -3,43,Eureka -4,45.7667,South Columbia -5,48.5,North Columbia - US/CAN border -6,55,Canada \ No newline at end of file diff --git a/recipes_nrt/deploy/deploy_cloud_echogram_viz_only.yaml b/recipes_nrt/deploy/deploy_cloud_echogram_viz_only.yaml new file mode 100644 index 0000000..3554a6c --- /dev/null +++ b/recipes_nrt/deploy/deploy_cloud_echogram_viz_only.yaml @@ -0,0 +1,15 @@ +flow_start_time: null + +source: + mode: git + git: + url: https://github.com/OSOceanAcoustics/echodataflow.git + branch: main + +default_work_pool_name: local +flows: + update_cache_MVBS: + deployment_name: update-cache-MVBS + interval: 10 + cron_offset: 3 + inject_time_offset: true diff --git a/recipes_nrt/deploy/deploy_cloud_echogram_viz_only_2vessels.yaml b/recipes_nrt/deploy/deploy_cloud_echogram_viz_only_2vessels.yaml new file mode 100644 index 0000000..6fb95e4 --- /dev/null +++ b/recipes_nrt/deploy/deploy_cloud_echogram_viz_only_2vessels.yaml @@ -0,0 +1,22 @@ +flow_start_time: null + +source: + mode: git + git: + url: https://github.com/OSOceanAcoustics/echodataflow.git + branch: main + +default_work_pool_name: local +flows: + update_cache_MVBS_vessel_1: + flow: update_cache_MVBS + deployment_name: update-cache-MVBS-vessel-1 + interval: 10 + cron_offset: 3 + inject_time_offset: true + update_cache_MVBS_vessel_2: + flow: update_cache_MVBS + deployment_name: update-cache-MVBS-vessel-2 + interval: 10 + cron_offset: 3 + inject_time_offset: true diff --git a/recipes_nrt/deploy/deploy_cloud_nrt.yaml b/recipes_nrt/deploy/deploy_cloud_nrt.yaml new file mode 100644 index 0000000..2b94fbb --- /dev/null +++ b/recipes_nrt/deploy/deploy_cloud_nrt.yaml @@ -0,0 +1,35 @@ +flow_start_time: # use ISO8601 format: if UTC use +00:00 or Z + +source: + mode: local + git: + url: https://github.com/echostack-org/echodataflow.git + branch: origin/main + +default_work_pool_name: cloud-local + +flows: + ingest_haul: + deployment_name: ingest-haul + interval: 5 + + ingest_NASC: + deployment_name: ingest-NASC + deployment_concurrency: + limit: 1 + collision_strategy: CANCEL_NEW + interval: 2 + + update_grid: + deployment_name: update-grid + triggers: + - expect: prefect.flow-run.Completed + resource_name: ingest-haul + - expect: prefect.flow-run.Completed + resource_name: ingest-NASC + + update_cache_MVBS: + deployment_name: update-cache-MVBS + interval: 10 + cron_offset: 3 + inject_time_offset: true diff --git a/recipes/deploy/deploy_cps_test.yaml b/recipes_nrt/deploy/deploy_cps_test.yaml similarity index 100% rename from recipes/deploy/deploy_cps_test.yaml rename to recipes_nrt/deploy/deploy_cps_test.yaml diff --git a/recipes_nrt/deploy/deploy_edge_nrt.yaml b/recipes_nrt/deploy/deploy_edge_nrt.yaml new file mode 100644 index 0000000..9d244f5 --- /dev/null +++ b/recipes_nrt/deploy/deploy_edge_nrt.yaml @@ -0,0 +1,34 @@ +flow_start_time: # use ISO8601 format: if UTC use +00:00 or Z + +source: + mode: local + +default_work_pool_name: edge-local + +flows: + raw2Sv: + deployment_name: raw2Sv + deployment_concurrency: + limit: 1 + collision_strategy: CANCEL_NEW + interval: 5 + + create_MVBS: + deployment_name: create-MVBS + interval: 10 + cron_offset: 3 + + predict_hake: + deployment_name: predict-hake + interval: 20 + + file_upload_acoustics: + deployment_name: file-upload-acoustics + flow: file_upload + interval: 10 + cron_offset: 3 + + file_upload_trawl: + deployment_name: file-upload-trawl + flow: file_upload + interval: 10 diff --git a/recipes_nrt/params/params_cloud_echogram_viz_only.yaml b/recipes_nrt/params/params_cloud_echogram_viz_only.yaml new file mode 100644 index 0000000..51758d3 --- /dev/null +++ b/recipes_nrt/params/params_cloud_echogram_viz_only.yaml @@ -0,0 +1,11 @@ +init: + counter_raw_copy: 0 + +flows: + update_cache_MVBS: + slice_mins: 180 + path_cache: "PATH/TO/CACHE" + path_MVBS: "BUCKET/FOLDER/acoustics/MVBS" + cred_file: "PATH/TO/.config/rclone/rclone.conf" + file_MVBS_csv: MVBS_files.csv + file_MVBS_zarr: latest_MVBS.zarr diff --git a/recipes_nrt/params/params_cloud_echogram_viz_only_2vessels.yaml b/recipes_nrt/params/params_cloud_echogram_viz_only_2vessels.yaml new file mode 100644 index 0000000..081dfaa --- /dev/null +++ b/recipes_nrt/params/params_cloud_echogram_viz_only_2vessels.yaml @@ -0,0 +1,18 @@ +init: + counter_raw_copy: 0 + +flows: + update_cache_MVBS_vessel_1: + slice_mins: 180 + path_cache: "PATH/TO/CACHE" + path_MVBS: "BUCKET/FOLDER/acoustics/MVBS" + cred_file: "PATH/TO/.config/rclone/rclone.conf" + file_MVBS_csv: MVBS_files.csv + file_MVBS_zarr: latest_MVBS.zarr + update_cache_MVBS_vessel_2: + slice_mins: 180 + path_cache: "PATH/TO/CACHE" + path_MVBS: "BUCKET/FOLDER/acoustics/MVBS" + cred_file: "PATH/TO/.config/rclone/rclone.conf" + file_MVBS_csv: MVBS_files.csv + file_MVBS_zarr: latest_MVBS.zarr diff --git a/recipes_nrt/params/params_cloud_nrt.yaml b/recipes_nrt/params/params_cloud_nrt.yaml new file mode 100644 index 0000000..9e1f306 --- /dev/null +++ b/recipes_nrt/params/params_cloud_nrt.yaml @@ -0,0 +1,34 @@ +flows: + ingest_haul: + path_vm_local: PATH/TO/DATA # path to local integrated data + path_bio_files: BUCKET/trawl # path to cloud trawl data + cred_file: PATH/TO/.config/rclone/rclone.conf + file_haul_info_all: haul_info_all.csv + file_specimen_all: specimen_all.csv + file_length_all: length_all.csv + file_length_count_all: length_count_all.csv + file_stratum_mean: stratum_mean.csv + path_stratum_def: PATH/TO/resources/inpfc_def.csv + + ingest_NASC: + path_vm_local: PATH/TO/DATA # path to local integrated data + path_NASC_files: BUCKET/acoustics/NASC # path to cloud acoustic data + cred_file: "PATH/TO/.config/rclone/rclone.conf" + file_NASC_all: NASC_all.csv + file_stratum_mean: stratum_mean.csv + file_NASC_all_grid: NASC_all_griddify.geojson + num_NASC_reprocess: 5 + new_file_num_limit: 200 + + update_grid: + path_vm_local: PATH/TO/DATA # path to local integrated data + file_NASC_all_grid: NASC_all_griddify.geojson + file_stratum_mean: stratum_mean.csv + + update_cache_MVBS: + slice_mins: 180 + path_cache: "PATH/TO/CACHE" + path_MVBS: "BUCKET/FOLDER/acoustics/MVBS" + cred_file: "PATH/TO/.config/rclone/rclone.conf" + file_MVBS_csv: MVBS_files.csv + file_MVBS_zarr: latest_MVBS.zarr diff --git a/recipes/params/params_cps_test.yaml b/recipes_nrt/params/params_cps_test.yaml similarity index 100% rename from recipes/params/params_cps_test.yaml rename to recipes_nrt/params/params_cps_test.yaml diff --git a/recipes_nrt/params/params_edge_nrt.yaml b/recipes_nrt/params/params_edge_nrt.yaml new file mode 100644 index 0000000..b22e049 --- /dev/null +++ b/recipes_nrt/params/params_edge_nrt.yaml @@ -0,0 +1,48 @@ +flows: + raw2Sv: + exclude_before: "YYYY-mm-DDTHH:MM:SS+00:00" + exclude_raw_file: [] + parallel: False + encode_mode: power + waveform_mode: CW + depth_offset: 7 + sonar_model: EK80 + nmea_sentence: GGA + filename_pattern: "PREFIX-*.raw" + path_raw: /PATH/TO/raw + path_main: /PATH/TO/local_main + file_Sv_csv: Sv_files.csv + + create_MVBS: + slice_mins: 20 + num_slices: 3 + range_bin: 1m + ping_time_bin: 5s + path_main: /PATH/TO/local_main + file_Sv_csv: Sv_files.csv + file_MVBS_csv: MVBS_files.csv + + predict_hake: + slice_mins: 40 + num_slices: 3 + temperature: 0.5 + softmax_threshold: 0.5 + max_depth: 590.0 + path_weight: /PATH/TO/model_weights/model.ckpt + path_main: /PATH/TO/local_main + file_MVBS_csv: MVBS_files.csv + file_prediction_csv: prediction_files.csv + + file_upload_acoustics: + src_dir: /PATH/TO/local_main + dest_dir: ENDPOINT:BUCKET/acoustics + exclude_subdirs: + - Sv + - raw + max_age: 2 + + file_upload_trawl: + src_dir: /PATH/TO/trawl + dest_dir: ENDPOINT:BUCKET/trawl + exclude_subdirs: [] + max_age: 2 diff --git a/recipes_postproc/deploy/deploy_postproc_edge.yaml b/recipes_postproc/deploy/deploy_postproc_edge.yaml new file mode 100644 index 0000000..0f23094 --- /dev/null +++ b/recipes_postproc/deploy/deploy_postproc_edge.yaml @@ -0,0 +1,42 @@ +source: + mode: local + +default_work_pool_name: local + +concurrency_groups: + raw2Sv: + limit: 1 + downstream: + limit: 3 + +flows: + raw2Sv_postprocessing: + deployment_name: raw2Sv-postprocessing + concurrency_limit: + limit: 1 + collision_strategy: CANCEL_NEW + concurrency_group: raw2Sv + task_runner: + type: dask + cluster_kwargs: + n_workers: 5 + threads_per_worker: 2 + processes: true + interval: 1 + + create_MVBS_postprocessing: + deployment_name: create-MVBS-postprocessing + concurrency_limit: + limit: 1 + collision_strategy: CANCEL_NEW + concurrency_group: downstream + interval: 1 + + predict_hake_postprocessing: + deployment_name: predict-hake-postprocessing + concurrency_limit: + limit: 1 + collision_strategy: CANCEL_NEW + concurrency_group: downstream + interval: 5 + cron_offset: 1 diff --git a/recipes_postproc/params/params_postproc_edge.yaml b/recipes_postproc/params/params_postproc_edge.yaml new file mode 100644 index 0000000..5ee5d76 --- /dev/null +++ b/recipes_postproc/params/params_postproc_edge.yaml @@ -0,0 +1,39 @@ +flows: + raw2Sv_postprocessing: + path_raw_list: /PATH/TO/SH2506_raw_list_organized.csv + path_raw: /PATH/TO/RAW_DATA + path_main: /PATH/TO/WORKFLOW_OUTPUT + start_time: "YYYY-mm-DDTHH:MM:SSZ" + end_time: "YYYY-mm-DDTHH:MM:SSZ" + new_file_num_limit: 200 + task_retries: 2 + task_retry_delay_seconds: 30 + max_flow_run_attempts: 2 + encode_mode: power + waveform_mode: CW + depth_offset: 7 + sonar_model: EK80 + nmea_sentence: GGA + file_Sv_csv: Sv_files.csv + + create_MVBS_postprocessing: + path_main: /PATH/TO/WORKFLOW_OUTPUT + new_file_num_limit: 20 + max_flow_run_attempts: 2 + slice_mins: 20 + range_bin: 1m + ping_time_bin: 5s + file_Sv_csv: Sv_files.csv + file_MVBS_csv: MVBS_files.csv + + predict_hake_postprocessing: + path_main: /PATH/TO/WORKFLOW_OUTPUT + path_weight: /PATH/TO/ML_MODEL/model.ckpt + prediction_slice_mins: 40 + temperature: 0.5 + softmax_threshold: 0.5 + max_depth: 590.0 + nasc_range_bin: 10m + nasc_dist_bin: 0.5nmi + file_MVBS_csv: MVBS_files.csv + file_prediction_csv: prediction_files.csv