From af3d6ba8d8f406d11bc3f0ddeea19dd087b4b322 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 5 Mar 2026 11:13:53 -0500 Subject: [PATCH 01/18] Separate each fetch into its own task --- src/swell/suites/3dvar/flow.cylc | 8 +- src/swell/tasks/get_observations.py | 324 ++++++++++++++-------------- 2 files changed, 170 insertions(+), 162 deletions(-) diff --git a/src/swell/suites/3dvar/flow.cylc b/src/swell/suites/3dvar/flow.cylc index 9b1d063fd..bf82b6413 100644 --- a/src/swell/suites/3dvar/flow.cylc +++ b/src/swell/suites/3dvar/flow.cylc @@ -60,7 +60,9 @@ GetBackground-{{model_component}} => RunJediVariationalExecutable-{{model_component}} GenerateBClimatology-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + {% for observation in models[model_component]['observations'] %} GetObservations-{{model_component}} => RenderJediObservations-{{model_component}} + {% endfor %} RenderJediObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} # EvaObservations @@ -125,8 +127,12 @@ [[ GetBackground-{{model_component}} ]] script = "swell task GetBackground $config -d $datetime -m {{model_component}}" - [[GetObservations-{{model_component}}]] + {% for observation in models[model_component]['observations'] %} + [[GetObservations-{{observation}}-{{model_component}}]] script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + [[[environment]]] + OBSERVATION = {{observation}} + {% endfor %} [[GenerateBClimatology-{{model_component}}]] script = "swell task GenerateBClimatology $config -d $datetime -m {{model_component}}" diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index f8c561450..4c2194b18 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -98,11 +98,17 @@ def execute(self) -> None: "tlapse" files need to be fetched. """ + # Get the observation from the environment + # ---------------------------------------- + observation = os.environ.get('OBSERVATION') + + if observation is None: + raise Exception('No observation specified.') + # Parse config # ------------ obs_experiment = self.config.obs_experiment() background_time_offset = self.config.background_time_offset() - observations = self.config.observations() window_length = self.config.window_length() crtm_coeff_dir = self.config.crtm_coeff_dir(None) window_length = self.config.window_length() @@ -147,180 +153,176 @@ def execute(self) -> None: # Read observation ioda names ioda_names_list = get_ioda_names_list() - # Loop over observation operators - # ------------------------------- - for observation in observations: - - # Open the observation operator dictionary - # ---------------------------------------- - observation_dict = self.jedi_rendering.render_interface_observations(observation) - - # Get the set obs providers for each observation - # ---------------------------------------------- - obs_provider = get_provider_for_observation(observation, ioda_names_list, self.logger) - - # Fetch observation files - # ----------------------- - combine_input_files = [] - # Here, we are fetching - for obs_num, obs_time in enumerate(obs_list_dto): - obs_window_begin = dt.strftime(obs_time, datetime_formats['iso_format']) - target_file = os.path.join(self.cycle_dir(), f'{observation}.{obs_num}.nc4') - combine_input_files.append(target_file) - - fetch_criteria = { - 'item': 'observation', # Required for r2d2 v3 - 'provider': obs_provider, # What we registered with - 'observation_type': observation, # From filename - 'file_extension': 'nc4', - 'window_start': obs_window_begin, # From filename timestamp - 'window_length': obs_window_length, # From filename - 'target_file': target_file, # Where to save - } - - try: - r2d2.fetch(**fetch_criteria) - self.logger.info(f"Successfully fetched {target_file}") - except Exception: - self.logger.info( - f"Failed to fetch {target_file}. " - "Fetch empty observation instead." - ) - - # fetch empty obs - r2d2.fetch( - item='observation', - provider='empty_provider', - observation_type='empty_type', - file_extension='nc4', - window_start='19700101T030000Z', - window_length='PT6H', - target_file=target_file, - ) - - # Check how many of the combine_input_files exist in the cycle directory. - # If all of them are missing proceed without creating an observation input - # file since bias correction files still need to be propagated to the next cycle - # for cycling VarBC. - # ----------------------------------------------------------------------- - if not any([os.path.exists(f) for f in combine_input_files]): - self.logger.info(f'None of the {observation} files exist for this cycle!') - else: - jedi_obs_file = observation_dict['obs space']['obsdatain']['engine']['obsfile'] - self.logger.info(f'Processing observation file {jedi_obs_file}') - # If obs_list_dto has one member, then just rename the file - # --------------------------------------------------------- - if len(obs_list_dto) == 1: - os.rename(combine_input_files[0], jedi_obs_file) - else: - self.read_and_combine(combine_input_files, jedi_obs_file) - # Change permission - os.chmod(jedi_obs_file, 0o644) - - # Otherwise there is only work to do if the observation operator has bias correction - # ---------------------------------------------------------------------------------- - if 'obs bias' not in observation_dict: - continue + # Open the observation operator dictionary + # ---------------------------------------- + observation_dict = self.jedi_rendering.render_interface_observations(observation) + + # Get the set obs providers for each observation + # ---------------------------------------------- + obs_provider = get_provider_for_observation(observation, ioda_names_list, self.logger) + + # Fetch observation files + # ----------------------- + combine_input_files = [] + # Here, we are fetching + for obs_num, obs_time in enumerate(obs_list_dto): + obs_window_begin = dt.strftime(obs_time, datetime_formats['iso_format']) + target_file = os.path.join(self.cycle_dir(), f'{observation}.{obs_num}.nc4') + combine_input_files.append(target_file) + + fetch_criteria = { + 'item': 'observation', # Required for r2d2 v3 + 'provider': obs_provider, # What we registered with + 'observation_type': observation, # From filename + 'file_extension': 'nc4', + 'window_start': obs_window_begin, # From filename timestamp + 'window_length': obs_window_length, # From filename + 'target_file': target_file, # Where to save + } + + try: + r2d2.fetch(**fetch_criteria) + self.logger.info(f"Successfully fetched {target_file}") + except Exception: + self.logger.info( + f"Failed to fetch {target_file}. " + "Fetch empty observation instead." + ) + + # fetch empty obs + r2d2.fetch( + item='observation', + provider='empty_provider', + observation_type='empty_type', + file_extension='nc4', + window_start='19700101T030000Z', + window_length='PT6H', + target_file=target_file, + ) - # Satellite and aircraft bias correction (coeff and cov) files - # ----------------------------------------------- - target_bccoef = observation_dict['obs bias']['input file'] - target_bccovr = observation_dict['obs bias']['covariance']['prior']['input file'] - - # We assume fetch is required unless we are cycling VarBC - fetch_required = True - - if cycling_varbc: - if self.cycle_time_dto() == self.start_cycle_point_dto(): - self.logger.info(f'Process bias file {target_bccoef} for the first cycle') - self.logger.info(f'Process bias file {target_bccovr} for the first cycle') - else: - self.logger.info(f'Using bias files from the previous cycle') - previous_bias_coef = self.previous_cycle_bias(target_bccoef, window_length) - previous_bias_covr = self.previous_cycle_bias(target_bccovr, window_length) - # Link the previous bias file to the current cycle directory - self.logger.info(f'Linking {previous_bias_coef} to {target_bccoef}') - self.geos.linker(previous_bias_coef, target_bccoef, dst_dir=self.cycle_dir()) - self.logger.info(f'Linking {previous_bias_covr} to {target_bccovr}') - self.geos.linker(previous_bias_covr, target_bccovr, dst_dir=self.cycle_dir()) - fetch_required = False - - # Determine the bias file extension and map to R2D2 file_type enum - if observation == 'aircraft_temperature': - bias_file_ext = 'acftbias' - bias_file_type = 'obsbias_coefficients' # Official JCSDA enum - bias_err_type = 'obsbias_coeff_errors' # Official JCSDA enum - # TODO: Do we want to use bias corrections for winds? - # TODO: Confirm for extension and err_type. Bias files exist for aircraft_wind - elif observation == 'aircraft_wind': - bias_file_type = None # Option A: Skip - # Option B: Enable (if bias files should be used for aircraft_wind) - # bias_file_ext = 'acftbias' - # bias_coef_type = 'obsbias_coefficients' - # bias_err_type = 'obsbias_coeff_errors' + # Check how many of the combine_input_files exist in the cycle directory. + # If all of them are missing proceed without creating an observation input + # file since bias correction files still need to be propagated to the next cycle + # for cycling VarBC. + # ----------------------------------------------------------------------- + if not any([os.path.exists(f) for f in combine_input_files]): + self.logger.info(f'None of the {observation} files exist for this cycle!') + else: + jedi_obs_file = observation_dict['obs space']['obsdatain']['engine']['obsfile'] + self.logger.info(f'Processing observation file {jedi_obs_file}') + # If obs_list_dto has one member, then just rename the file + # --------------------------------------------------------- + if len(obs_list_dto) == 1: + os.rename(combine_input_files[0], jedi_obs_file) else: - # Satellite observations - bias_file_ext = 'satbias' - bias_file_type = 'satbias' # Official JCSDA enum - bias_err_type = 'obsbias_coeff_errors' # Official JCSDA enum - - # This will skip the fetch if we are cycling VarBC - if bias_file_type is not None: - if fetch_required: - # Fetch coefficients file (.acftbias or .satbias) - self.logger.info(f'Processing bias file {target_bccoef}') - r2d2.fetch( - item='bias_correction', - target_file=target_bccoef, - model=r2d2_model, - experiment=obs_experiment, - provider='gsi', - observation_type=observation, - file_extension=bias_file_ext, - file_type=bias_file_type, - date=background_time_iso - ) - - r2d2.fetch( - item='bias_correction', - target_file=target_bccovr, - model=r2d2_model, - experiment=obs_experiment, - provider='gsi', - observation_type=observation, - file_extension=bias_file_ext + '_cov', - file_type=bias_err_type, # obsbias_coeff_errors Official JCSDA enum - date=background_time_iso - ) - # Change permission - os.chmod(target_bccoef, 0o644) - os.chmod(target_bccovr, 0o644) - - # Skip time lapse part for aircraft observations - # ---------------------------------------------- - if observation == 'aircraft_temperature' or observation == 'aircraft_wind': - continue + self.read_and_combine(combine_input_files, jedi_obs_file) + # Change permission + os.chmod(jedi_obs_file, 0o644) + + # Otherwise there is only work to do if the observation operator has bias correction + # ---------------------------------------------------------------------------------- + if 'obs bias' not in observation_dict: + return - # Satellite time lapse - # -------------------- - for target_file in self.get_tlapse_files(observation_dict): + # Satellite and aircraft bias correction (coeff and cov) files + # ----------------------------------------------- + target_bccoef = observation_dict['obs bias']['input file'] + target_bccovr = observation_dict['obs bias']['covariance']['prior']['input file'] - self.logger.info(f'Processing satellite time lapse file {target_file}') + # We assume fetch is required unless we are cycling VarBC + fetch_required = True + if cycling_varbc: + if self.cycle_time_dto() == self.start_cycle_point_dto(): + self.logger.info(f'Process bias file {target_bccoef} for the first cycle') + self.logger.info(f'Process bias file {target_bccovr} for the first cycle') + else: + self.logger.info(f'Using bias files from the previous cycle') + previous_bias_coef = self.previous_cycle_bias(target_bccoef, window_length) + previous_bias_covr = self.previous_cycle_bias(target_bccovr, window_length) + # Link the previous bias file to the current cycle directory + self.logger.info(f'Linking {previous_bias_coef} to {target_bccoef}') + self.geos.linker(previous_bias_coef, target_bccoef, dst_dir=self.cycle_dir()) + self.logger.info(f'Linking {previous_bias_covr} to {target_bccovr}') + self.geos.linker(previous_bias_covr, target_bccovr, dst_dir=self.cycle_dir()) + fetch_required = False + + # Determine the bias file extension and map to R2D2 file_type enum + if observation == 'aircraft_temperature': + bias_file_ext = 'acftbias' + bias_file_type = 'obsbias_coefficients' # Official JCSDA enum + bias_err_type = 'obsbias_coeff_errors' # Official JCSDA enum + # TODO: Do we want to use bias corrections for winds? + # TODO: Confirm for extension and err_type. Bias files exist for aircraft_wind + elif observation == 'aircraft_wind': + bias_file_type = None # Option A: Skip + # Option B: Enable (if bias files should be used for aircraft_wind) + # bias_file_ext = 'acftbias' + # bias_coef_type = 'obsbias_coefficients' + # bias_err_type = 'obsbias_coeff_errors' + else: + # Satellite observations + bias_file_ext = 'satbias' + bias_file_type = 'satbias' # Official JCSDA enum + bias_err_type = 'obsbias_coeff_errors' # Official JCSDA enum + + # This will skip the fetch if we are cycling VarBC + if bias_file_type is not None: + if fetch_required: + # Fetch coefficients file (.acftbias or .satbias) + self.logger.info(f'Processing bias file {target_bccoef}') r2d2.fetch( item='bias_correction', - target_file=target_file, + target_file=target_bccoef, model=r2d2_model, experiment=obs_experiment, provider='gsi', observation_type=observation, - file_extension='tlapse', - file_type='obsbias_tlapse', # Official JCSDA enum + file_extension=bias_file_ext, + file_type=bias_file_type, date=background_time_iso ) - # Change permission - os.chmod(target_file, 0o644) + r2d2.fetch( + item='bias_correction', + target_file=target_bccovr, + model=r2d2_model, + experiment=obs_experiment, + provider='gsi', + observation_type=observation, + file_extension=bias_file_ext + '_cov', + file_type=bias_err_type, # obsbias_coeff_errors Official JCSDA enum + date=background_time_iso + ) + # Change permission + os.chmod(target_bccoef, 0o644) + os.chmod(target_bccovr, 0o644) + + # Skip time lapse part for aircraft observations + # ---------------------------------------------- + if observation == 'aircraft_temperature' or observation == 'aircraft_wind': + return + + # Satellite time lapse + # -------------------- + for target_file in self.get_tlapse_files(observation_dict): + + self.logger.info(f'Processing satellite time lapse file {target_file}') + + r2d2.fetch( + item='bias_correction', + target_file=target_file, + model=r2d2_model, + experiment=obs_experiment, + provider='gsi', + observation_type=observation, + file_extension='tlapse', + file_type='obsbias_tlapse', # Official JCSDA enum + date=background_time_iso + ) + + # Change permission + os.chmod(target_file, 0o644) # ---------------------------------------------------------------------------------------------- From e616d67cbfba269443b4281591166b63a6aa809b Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 5 Mar 2026 11:23:26 -0500 Subject: [PATCH 02/18] Fix cylc templating --- src/swell/suites/3dvar/flow.cylc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/swell/suites/3dvar/flow.cylc b/src/swell/suites/3dvar/flow.cylc index bf82b6413..08b94f281 100644 --- a/src/swell/suites/3dvar/flow.cylc +++ b/src/swell/suites/3dvar/flow.cylc @@ -61,7 +61,7 @@ GenerateBClimatology-{{model_component}} => RunJediVariationalExecutable-{{model_component}} {% for observation in models[model_component]['observations'] %} - GetObservations-{{model_component}} => RenderJediObservations-{{model_component}} + GetObservations-{{observation}}-{{model_component}} => RenderJediObservations-{{model_component}} {% endfor %} RenderJediObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} @@ -127,12 +127,15 @@ [[ GetBackground-{{model_component}} ]] script = "swell task GetBackground $config -d $datetime -m {{model_component}}" + # --------------------------------------------------------------------------------------------- + {% for observation in models[model_component]['observations'] %} [[GetObservations-{{observation}}-{{model_component}}]] script = "swell task GetObservations $config -d $datetime -m {{model_component}}" [[[environment]]] OBSERVATION = {{observation}} {% endfor %} + # --------------------------------------------------------------------------------------------- [[GenerateBClimatology-{{model_component}}]] script = "swell task GenerateBClimatology $config -d $datetime -m {{model_component}}" From 12ed178b539e12699df59c69b9933aa50e97dd9e Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 5 Mar 2026 14:26:22 -0500 Subject: [PATCH 03/18] Fix task file parsing --- src/swell/suites/3dvar/flow.cylc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/swell/suites/3dvar/flow.cylc b/src/swell/suites/3dvar/flow.cylc index 08b94f281..2521d5eca 100644 --- a/src/swell/suites/3dvar/flow.cylc +++ b/src/swell/suites/3dvar/flow.cylc @@ -21,6 +21,9 @@ initial cycle point = {{start_cycle_point}} final cycle point = {{final_cycle_point}} runahead limit = {{runahead_limit}} + [[queues]] + [[[default]]] + limit=6 [[graph]] R1 = """ @@ -128,6 +131,10 @@ script = "swell task GetBackground $config -d $datetime -m {{model_component}}" # --------------------------------------------------------------------------------------------- + # [[GetObservations-{{model_component}}]] + # script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + # --------------------------------------------------------------------------------------------- + {% for observation in models[model_component]['observations'] %} [[GetObservations-{{observation}}-{{model_component}}]] From 7ee41290f614e4b86bdb55b6beebee016e8d4d67 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 5 Mar 2026 14:38:31 -0500 Subject: [PATCH 04/18] Up to 3dvar_atmos --- src/swell/suites/3dvar/flow.cylc | 2 +- src/swell/suites/3dvar_atmos/flow.cylc | 31 +++++++++++++++++++--- src/swell/suites/3dvar_cycle/flow.cylc | 16 ++++++++++- src/swell/suites/hofx/flow.cylc | 16 ++++++++++- src/swell/suites/hofx_cf/flow.cylc | 16 ++++++++++- src/swell/suites/localensembleda/flow.cylc | 16 ++++++++++- 6 files changed, 88 insertions(+), 9 deletions(-) diff --git a/src/swell/suites/3dvar/flow.cylc b/src/swell/suites/3dvar/flow.cylc index 2521d5eca..13b294187 100644 --- a/src/swell/suites/3dvar/flow.cylc +++ b/src/swell/suites/3dvar/flow.cylc @@ -131,11 +131,11 @@ script = "swell task GetBackground $config -d $datetime -m {{model_component}}" # --------------------------------------------------------------------------------------------- + # This has to be here so swell will successfully register the task # [[GetObservations-{{model_component}}]] # script = "swell task GetObservations $config -d $datetime -m {{model_component}}" # --------------------------------------------------------------------------------------------- - {% for observation in models[model_component]['observations'] %} [[GetObservations-{{observation}}-{{model_component}}]] script = "swell task GetObservations $config -d $datetime -m {{model_component}}" diff --git a/src/swell/suites/3dvar_atmos/flow.cylc b/src/swell/suites/3dvar_atmos/flow.cylc index e0861d645..9e89067bf 100644 --- a/src/swell/suites/3dvar_atmos/flow.cylc +++ b/src/swell/suites/3dvar_atmos/flow.cylc @@ -13,6 +13,9 @@ [scheduler] UTC mode = True allow implicit tasks = False + [[queues]] + [[[default]]] + limit=6 # -------------------------------------------------------------------------------------------------- @@ -58,11 +61,11 @@ {% if cycling_varbc %} # Cycling VarBC is active, biases from the previous cycle will be used - RunJediVariationalExecutable-{{model_component}}[-PT6H] => GetObservations-{{model_component}} + RunJediVariationalExecutable-{{model_component}}[-PT6H] => GetObservationsStart {% else %} # Cycling VarBC is inactive, static bias files will be used - GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} + GetObsNotInR2d2-{{model_component}}: fail? => GetObservationsStart {% endif %} # Perform staging that is cycle dependent @@ -73,7 +76,10 @@ CloneJedi[^] => StageJediCycle-{{model_component}} StageJediCycle-{{model_component}} => RunJediVariationalExecutable-{{model_component}} GetBackgroundGeosExperiment-{{model_component}}? | GetBackground-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => RenderJediObservations-{{model_component}} + {% for observation in models[model_component]['observations'] %} + GetObservationsStart => GetObservations-{{observation}}-{{model_component}} => GetObservationsFinish + {% endfor %} + GetObsNotInR2d2-{{model_component}}? | GetObservationsFinish => RenderJediObservations-{{model_component}} RenderJediObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} GenerateObservingSystemRecords-{{model_component}} => RunJediVariationalExecutable-{{model_component}} @@ -129,6 +135,12 @@ --{{key}} = {{value}} {%- endfor %} + [[GetObservationsStart]] + script = "true" + + [[GetObservationsFinish]] + script = "true" + {% for model_component in model_components %} [[CloneGeosMksi-{{model_component}}]] @@ -149,8 +161,19 @@ [[GetBackgroundGeosExperiment-{{model_component}} ]] script = "swell task GetBackgroundGeosExperiment $config -d $datetime -m {{model_component}}" - [[GetObservations-{{model_component}}]] + # --------------------------------------------------------------------------------------------- + # This has to be here so swell will successfully register the task + # [[GetObservations-{{model_component}}]] + # script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + # --------------------------------------------------------------------------------------------- + + {% for observation in models[model_component]['observations'] %} + [[GetObservations-{{observation}}-{{model_component}}]] script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + [[[environment]]] + OBSERVATION = {{observation}} + {% endfor %} + # --------------------------------------------------------------------------------------------- [[RenderJediObservations-{{model_component}}]] script = "swell task RenderJediObservations $config -d $datetime -m {{model_component}}" diff --git a/src/swell/suites/3dvar_cycle/flow.cylc b/src/swell/suites/3dvar_cycle/flow.cylc index 8db2144fd..7b315cfbd 100644 --- a/src/swell/suites/3dvar_cycle/flow.cylc +++ b/src/swell/suites/3dvar_cycle/flow.cylc @@ -13,6 +13,9 @@ [scheduler] UTC mode = True allow implicit tasks = False + [[queues]] + [[[default]]] + limit=6 # -------------------------------------------------------------------------------------------------- @@ -198,8 +201,19 @@ [[StageJediCycle-{{model_component}}]] script = "swell task StageJedi $config -d $datetime -m {{model_component}}" - [[GetObservations-{{model_component}}]] + # --------------------------------------------------------------------------------------------- + # This has to be here so swell will successfully register the task + # [[GetObservations-{{model_component}}]] + # script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + # --------------------------------------------------------------------------------------------- + + {% for observation in models[model_component]['observations'] %} + [[GetObservations-{{observation}}-{{model_component}}]] script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + [[[environment]]] + OBSERVATION = {{observation}} + {% endfor %} + # --------------------------------------------------------------------------------------------- [[GenerateBClimatology-{{model_component}}]] script = "swell task GenerateBClimatology $config -d $datetime -m {{model_component}}" diff --git a/src/swell/suites/hofx/flow.cylc b/src/swell/suites/hofx/flow.cylc index 666399b9d..d1f7c99e2 100644 --- a/src/swell/suites/hofx/flow.cylc +++ b/src/swell/suites/hofx/flow.cylc @@ -13,6 +13,9 @@ [scheduler] UTC mode = True allow implicit tasks = False + [[queues]] + [[[default]]] + limit=6 # -------------------------------------------------------------------------------------------------- @@ -131,8 +134,19 @@ [[GetBackgroundGeosExperiment-{{model_component}} ]] script = "swell task GetBackgroundGeosExperiment $config -d $datetime -m {{model_component}}" - [[GetObservations-{{model_component}}]] + # --------------------------------------------------------------------------------------------- + # This has to be here so swell will successfully register the task + # [[GetObservations-{{model_component}}]] + # script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + # --------------------------------------------------------------------------------------------- + + {% for observation in models[model_component]['observations'] %} + [[GetObservations-{{observation}}-{{model_component}}]] script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + [[[environment]]] + OBSERVATION = {{observation}} + {% endfor %} + # --------------------------------------------------------------------------------------------- [[GetObsNotInR2d2-{{model_component}}]] script = "swell task GetObsNotInR2d2 $config -d $datetime -m {{model_component}}" diff --git a/src/swell/suites/hofx_cf/flow.cylc b/src/swell/suites/hofx_cf/flow.cylc index 562482ecf..dc3d2e58e 100644 --- a/src/swell/suites/hofx_cf/flow.cylc +++ b/src/swell/suites/hofx_cf/flow.cylc @@ -13,6 +13,9 @@ [scheduler] UTC mode = True allow implicit tasks = False + [[queues]] + [[[default]]] + limit=6 # -------------------------------------------------------------------------------------------------- @@ -115,8 +118,19 @@ [[GetBackground-{{model_component}}]] script = "swell task GetBackground $config -d $datetime -m {{model_component}}" - [[GetObservations-{{model_component}}]] + # --------------------------------------------------------------------------------------------- + # This has to be here so swell will successfully register the task + # [[GetObservations-{{model_component}}]] + # script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + # --------------------------------------------------------------------------------------------- + + {% for observation in models[model_component]['observations'] %} + [[GetObservations-{{observation}}-{{model_component}}]] script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + [[[environment]]] + OBSERVATION = {{observation}} + {% endfor %} + # --------------------------------------------------------------------------------------------- [[RenderJediObservations-{{model_component}}]] script = "swell task RenderJediObservations $config -d $datetime -m {{model_component}}" diff --git a/src/swell/suites/localensembleda/flow.cylc b/src/swell/suites/localensembleda/flow.cylc index 8675bccba..409628324 100644 --- a/src/swell/suites/localensembleda/flow.cylc +++ b/src/swell/suites/localensembleda/flow.cylc @@ -13,6 +13,9 @@ [scheduler] UTC mode = True allow implicit tasks = False + [[queues]] + [[[default]]] + limit=6 # -------------------------------------------------------------------------------------------------- @@ -171,8 +174,19 @@ --{{key}} = {{value}} {%- endfor %} - [[GetObservations-{{model_component}}]] + # --------------------------------------------------------------------------------------------- + # This has to be here so swell will successfully register the task + # [[GetObservations-{{model_component}}]] + # script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + # --------------------------------------------------------------------------------------------- + + {% for observation in models[model_component]['observations'] %} + [[GetObservations-{{observation}}-{{model_component}}]] script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + [[[environment]]] + OBSERVATION = {{observation}} + {% endfor %} + # --------------------------------------------------------------------------------------------- [[GetObsNotInR2d2-{{model_component}}]] script = "swell task GetObsNotInR2d2 $config -d $datetime -m {{model_component}}" From 4ead142fae94379db5a1d89133cbe63473971221 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 5 Mar 2026 14:47:08 -0500 Subject: [PATCH 05/18] Fix queue --- src/swell/suites/3dvar_atmos/flow.cylc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/swell/suites/3dvar_atmos/flow.cylc b/src/swell/suites/3dvar_atmos/flow.cylc index 9e89067bf..a928f52d0 100644 --- a/src/swell/suites/3dvar_atmos/flow.cylc +++ b/src/swell/suites/3dvar_atmos/flow.cylc @@ -13,9 +13,6 @@ [scheduler] UTC mode = True allow implicit tasks = False - [[queues]] - [[[default]]] - limit=6 # -------------------------------------------------------------------------------------------------- @@ -24,6 +21,9 @@ initial cycle point = {{start_cycle_point}} final cycle point = {{final_cycle_point}} runahead limit = {{runahead_limit}} + [[queues]] + [[[default]]] + limit=6 [[graph]] R1 = """ From 6a83ca396d3cd9c75ffa4afbe7d1684db4953371 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 5 Mar 2026 15:01:56 -0500 Subject: [PATCH 06/18] Add all suites --- src/swell/suites/3dvar_cycle/flow.cylc | 10 ++++++---- src/swell/suites/hofx/flow.cylc | 19 ++++++++++++++----- src/swell/suites/hofx_cf/flow.cylc | 12 ++++++------ src/swell/suites/localensembleda/flow.cylc | 20 +++++++++++++++----- 4 files changed, 41 insertions(+), 20 deletions(-) diff --git a/src/swell/suites/3dvar_cycle/flow.cylc b/src/swell/suites/3dvar_cycle/flow.cylc index 7b315cfbd..7136f2887 100644 --- a/src/swell/suites/3dvar_cycle/flow.cylc +++ b/src/swell/suites/3dvar_cycle/flow.cylc @@ -13,9 +13,6 @@ [scheduler] UTC mode = True allow implicit tasks = False - [[queues]] - [[[default]]] - limit=6 # -------------------------------------------------------------------------------------------------- @@ -23,6 +20,9 @@ initial cycle point = {{start_cycle_point}} final cycle point = {{final_cycle_point}} + [[queues]] + [[[default]]] + limit=6 [[graph]] R1 = """ @@ -82,7 +82,9 @@ StageJediCycle-{{model_component}} => RunJediVariationalExecutable-{{model_component}} GenerateBClimatology-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - GetObservations-{{model_component}} => RenderJediObservations-{{model_component}} + {% for observation in models[model_component]['observations'] %} + GetObservations-{{observation}}-{{model_component}} => RenderJediObservations-{{model_component}} + {% endfor %} RenderJediObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} # Run analysis diagnostics diff --git a/src/swell/suites/hofx/flow.cylc b/src/swell/suites/hofx/flow.cylc index d1f7c99e2..e72499878 100644 --- a/src/swell/suites/hofx/flow.cylc +++ b/src/swell/suites/hofx/flow.cylc @@ -13,9 +13,6 @@ [scheduler] UTC mode = True allow implicit tasks = False - [[queues]] - [[[default]]] - limit=6 # -------------------------------------------------------------------------------------------------- @@ -24,6 +21,9 @@ initial cycle point = {{start_cycle_point}} final cycle point = {{final_cycle_point}} runahead limit = {{runahead_limit}} + [[queues]] + [[[default]]] + limit=6 [[graph]] R1 = """ @@ -58,7 +58,7 @@ GetBackgroundGeosExperiment-{{model_component}} :fail? => GetBackground-{{model_component}} # Get observations - GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} + GetObsNotInR2d2-{{model_component}}: fail? => GetObservationsStart # Perform staging that is cycle dependent StageJediCycle-{{model_component}} @@ -68,7 +68,10 @@ CloneJedi[^] => StageJediCycle-{{model_component}} StageJediCycle-{{model_component}} => RunJediHofxExecutable-{{model_component}} GetBackgroundGeosExperiment-{{model_component}}? | GetBackground-{{model_component}} => RunJediHofxExecutable-{{model_component}} - GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => RenderJediObservations-{{model_component}} + {% for observation in models[model_component]['observations'] %} + GetObservationsStart => GetObservations-{{observation}}-{{model_component}} => GetObservationsFinish + {% endfor %} + GetObsNotInR2d2-{{model_component}}? | GetObservationsFinish => RenderJediObservations-{{model_component}} RenderJediObservations-{{model_component}} => RunJediHofxExecutable-{{model_component}} GenerateObservingSystemRecords-{{model_component}} => RunJediHofxExecutable-{{model_component}} @@ -117,6 +120,12 @@ --{{key}} = {{value}} {%- endfor %} + [[GetObservationsStart]] + script = "true" + + [[GetObservationsFinish]] + script = "true" + {% for model_component in model_components %} [[CloneGeosMksi-{{model_component}}]] diff --git a/src/swell/suites/hofx_cf/flow.cylc b/src/swell/suites/hofx_cf/flow.cylc index dc3d2e58e..0dfac4c7b 100644 --- a/src/swell/suites/hofx_cf/flow.cylc +++ b/src/swell/suites/hofx_cf/flow.cylc @@ -13,9 +13,6 @@ [scheduler] UTC mode = True allow implicit tasks = False - [[queues]] - [[[default]]] - limit=6 # -------------------------------------------------------------------------------------------------- @@ -24,6 +21,9 @@ initial cycle point = {{start_cycle_point}} final cycle point = {{final_cycle_point}} runahead limit = {{runahead_limit}} + [[queues]] + [[[default]]] + limit=6 [[graph]] R1 = """ @@ -51,8 +51,9 @@ # Get background GetBackground-{{model_component}} - # Get observations - GetObservations-{{model_component}} + {% for observation in models[model_component]['observations'] %} + GetObservations-{{observation}}-{{model_component}} => RenderJediObservations-{{model_component}} + {% endfor %} # Perform staging that is cycle dependent StageJediCycle-{{model_component}} @@ -62,7 +63,6 @@ StageJediCycle-{{model_component}} => RunJediHofxExecutable-{{model_component}} GetBackground-{{model_component}} => RunJediHofxExecutable-{{model_component}} - GetObservations-{{model_component}} => RenderJediObservations-{{model_component}} RenderJediObservations-{{model_component}} => RunJediHofxExecutable-{{model_component}} # EvaObservations diff --git a/src/swell/suites/localensembleda/flow.cylc b/src/swell/suites/localensembleda/flow.cylc index 409628324..82e351c6d 100644 --- a/src/swell/suites/localensembleda/flow.cylc +++ b/src/swell/suites/localensembleda/flow.cylc @@ -13,9 +13,6 @@ [scheduler] UTC mode = True allow implicit tasks = False - [[queues]] - [[[default]]] - limit=6 # -------------------------------------------------------------------------------------------------- @@ -24,6 +21,9 @@ initial cycle point = {{start_cycle_point}} final cycle point = {{final_cycle_point}} runahead limit = {{runahead_limit}} + [[queues]] + [[[default]]] + limit=6 [[graph]] R1 = """ @@ -54,9 +54,13 @@ # Perform staging that is cycle dependent BuildJediByLinking[^]? | BuildJedi[^] => StageJediCycle-{{model_component}} => sync_point - GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} + GetObsNotInR2d2-{{model_component}}: fail? => GetObservationsStart + + {% for observation in models[model_component]['observations'] %} + GetObservationsStart => GetObservations-{{observation}}-{{model_component}} => GetObservationsFinish + {% endfor %} - GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => RenderJediObservations-{{model_component}} + GetObsNotInR2d2-{{model_component}}? | GetObservationsFinish => RenderJediObservations-{{model_component}} RenderJediObservations-{{model_component}} => sync_point @@ -136,6 +140,12 @@ --{{key}} = {{value}} {%- endfor %} + [[GetObservationsStart]] + script = "true" + + [[GetObservationsFinish]] + script = "true" + {% for model_component in model_components %} [[CloneGeosMksi-{{model_component}}]] From d99b12e777f60cd030ce737611732eb581448e12 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 5 Mar 2026 15:47:23 -0500 Subject: [PATCH 07/18] Fix bug in metop c yaml --- .../interfaces/geos_atmosphere/observations/iasi_metop-c.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml index 7912785f5..6a018adda 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml @@ -218,7 +218,7 @@ obs post filters: options: channels: *iasi_metop-c_channels error parameter vector: *iasi_metop-c_obserr - use_flag: &useflag_iasi_metop-c {{iasi_metopb_active_channels}} + use_flag: &useflag_iasi_metop-c {{iasi_metopc_active_channels}} use_flag_clddet: &clddet_iasi_metop-c [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, From a4a95a43526880ec2f04832735e10bece7ee7a01 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 5 Mar 2026 15:59:00 -0500 Subject: [PATCH 08/18] Handle observation specification using arguments --- src/swell/suites/3dvar/flow.cylc | 4 +--- src/swell/suites/3dvar_atmos/flow.cylc | 2 +- src/swell/suites/3dvar_cycle/flow.cylc | 2 +- src/swell/suites/hofx/flow.cylc | 2 +- src/swell/suites/hofx_cf/flow.cylc | 2 +- src/swell/swell.py | 8 +++++++- src/swell/tasks/base/task_base.py | 17 +++++++++++++++-- src/swell/tasks/get_observations.py | 5 +++-- 8 files changed, 30 insertions(+), 12 deletions(-) diff --git a/src/swell/suites/3dvar/flow.cylc b/src/swell/suites/3dvar/flow.cylc index 13b294187..67309d0dc 100644 --- a/src/swell/suites/3dvar/flow.cylc +++ b/src/swell/suites/3dvar/flow.cylc @@ -138,9 +138,7 @@ {% for observation in models[model_component]['observations'] %} [[GetObservations-{{observation}}-{{model_component}}]] - script = "swell task GetObservations $config -d $datetime -m {{model_component}}" - [[[environment]]] - OBSERVATION = {{observation}} + script = "swell task GetObservations $config -d $datetime -m {{model_component}} -a {{observation}}" {% endfor %} # --------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_atmos/flow.cylc b/src/swell/suites/3dvar_atmos/flow.cylc index a928f52d0..b648c7ea1 100644 --- a/src/swell/suites/3dvar_atmos/flow.cylc +++ b/src/swell/suites/3dvar_atmos/flow.cylc @@ -169,7 +169,7 @@ {% for observation in models[model_component]['observations'] %} [[GetObservations-{{observation}}-{{model_component}}]] - script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + script = "swell task GetObservations $config -d $datetime -m {{model_component}} -a {{observation}}" [[[environment]]] OBSERVATION = {{observation}} {% endfor %} diff --git a/src/swell/suites/3dvar_cycle/flow.cylc b/src/swell/suites/3dvar_cycle/flow.cylc index 7136f2887..a19417415 100644 --- a/src/swell/suites/3dvar_cycle/flow.cylc +++ b/src/swell/suites/3dvar_cycle/flow.cylc @@ -211,7 +211,7 @@ {% for observation in models[model_component]['observations'] %} [[GetObservations-{{observation}}-{{model_component}}]] - script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + script = "swell task GetObservations $config -d $datetime -m {{model_component}} -a {{observation}}" [[[environment]]] OBSERVATION = {{observation}} {% endfor %} diff --git a/src/swell/suites/hofx/flow.cylc b/src/swell/suites/hofx/flow.cylc index e72499878..dec8dddc2 100644 --- a/src/swell/suites/hofx/flow.cylc +++ b/src/swell/suites/hofx/flow.cylc @@ -151,7 +151,7 @@ {% for observation in models[model_component]['observations'] %} [[GetObservations-{{observation}}-{{model_component}}]] - script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + script = "swell task GetObservations $config -d $datetime -m {{model_component}} -a {{observation}}" [[[environment]]] OBSERVATION = {{observation}} {% endfor %} diff --git a/src/swell/suites/hofx_cf/flow.cylc b/src/swell/suites/hofx_cf/flow.cylc index 0dfac4c7b..5695cd301 100644 --- a/src/swell/suites/hofx_cf/flow.cylc +++ b/src/swell/suites/hofx_cf/flow.cylc @@ -121,7 +121,7 @@ # --------------------------------------------------------------------------------------------- # This has to be here so swell will successfully register the task # [[GetObservations-{{model_component}}]] - # script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + # script = "swell task GetObservations $config -d $datetime -m {{model_component}} -a {{observation}}" # --------------------------------------------------------------------------------------------- {% for observation in models[model_component]['observations'] %} diff --git a/src/swell/swell.py b/src/swell/swell.py index 944a43779..027aeee14 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -82,6 +82,10 @@ def swell_driver() -> None: or for task-model combinations. """ +additional_parameter_help = """ +Additional option to specify parameters to task, context-dependent on individual task. +""" + # -------------------------------------------------------------------------------------------------- @@ -183,12 +187,14 @@ def launch( @click.argument('config') @click.option('-d', '--datetime', 'datetime', default=None, help=datetime_help) @click.option('-m', '--model', 'model', default=None, help=model_help) +@click.option('-a', '--additional-parameter', 'additional_parameter', default=None, help=additional_parameter_help) @click.option('-p', '--ensemblePacket', 'ensemblePacket', default=None, help=ensemble_help) def task( task: str, config: str, datetime: Optional[str], model: Optional[str], + additional_parameter: Optional[str], ensemblePacket: Optional[str] ) -> None: """ @@ -201,7 +207,7 @@ def task( config (str): Path to the configuration file for the task.\n """ - task_wrapper(task, config, datetime, model, ensemblePacket) + task_wrapper(task, config, datetime, model, additional_parameter, ensemblePacket) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index f28de8466..fa2f2a868 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -41,6 +41,7 @@ def __init__( datetime_input: Optional[str], model: str, ensemblePacket: Optional[str], + additional_parameter: Optional[str], task_name: str ) -> None: @@ -62,6 +63,10 @@ def __init__( if datetime_input is not None: self.__datetime__ = Datetime(datetime_input) + # Keep copy of additional parameter + # --------------------------------- + self.__additional_parameter__ = additional_parameter + # Keep copy of ensemblePacket # --------------------------- self.__ensemble_packet__ = ensemblePacket @@ -172,6 +177,11 @@ def get_model(self) -> str: # ---------------------------------------------------------------------------------------------- + def get_parameter(self) -> str: + return self.__additional_parameter__ + + # ---------------------------------------------------------------------------------------------- + def get_model_components(self) -> Union[str, list]: return self.__model_components__ @@ -275,6 +285,7 @@ def create_task( config: str, datetime: Union[str, dt, None], model: str, + additional_parameter: Optional[str], ensemblePacket: Optional[str] ) -> taskBase: @@ -315,7 +326,7 @@ def create_task( factory_logger.info(f'Using module swell.tasks.{task_lower}') # Return task object - return task_class(config, datetime, model, ensemblePacket, task) + return task_class(config, datetime, model, ensemblePacket, additional_parameter, task) # -------------------------------------------------------------------------------------------------- @@ -346,13 +357,15 @@ def task_wrapper( config: str, datetime: Union[str, dt, None], model: Optional[str], + additional_parameter: Optional[str], ensemblePacket: Optional[str] ) -> None: # Create the object constrc_start = time.perf_counter() creator = taskFactory() - task_object = creator.create_task(task, config, datetime, model, ensemblePacket) + task_object = creator.create_task(task, config, datetime, model, additional_parameter, + ensemblePacket) constrc_final = time.perf_counter() constrc_time = f'Constructed in {constrc_final - constrc_start:0.4f} seconds' diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index 4c2194b18..f8675a6f7 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -100,10 +100,11 @@ def execute(self) -> None: # Get the observation from the environment # ---------------------------------------- - observation = os.environ.get('OBSERVATION') + observation = self.get_parameter() if observation is None: - raise Exception('No observation specified.') + raise Exception('No observation specified. Specify observation ' + 'using `swell task GetObservations -a ...`') # Parse config # ------------ From 121a1a51fa8279a6e2154e711f3f5fb42457e351 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 5 Mar 2026 16:06:26 -0500 Subject: [PATCH 09/18] add localensembleda and 3dfgat_cycle --- src/swell/suites/3dfgat_cycle/flow.cylc | 22 +++++++++++++++++++--- src/swell/suites/localensembleda/flow.cylc | 2 +- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/swell/suites/3dfgat_cycle/flow.cylc b/src/swell/suites/3dfgat_cycle/flow.cylc index db6b339ba..c4e82a4be 100644 --- a/src/swell/suites/3dfgat_cycle/flow.cylc +++ b/src/swell/suites/3dfgat_cycle/flow.cylc @@ -21,6 +21,9 @@ initial cycle point = {{start_cycle_point}} final cycle point = {{final_cycle_point}} runahead limit = {{runahead_limit}} + [[queues]] + [[[default]]] + limit=6 [[graph]] R1 = """ @@ -80,7 +83,9 @@ StageJediCycle-{{model_component}} => RunJediFgatExecutable-{{model_component}} GenerateBClimatology-{{model_component}} => RunJediFgatExecutable-{{model_component}} - GetObservations-{{model_component}} => RenderJediObservations-{{model_component}} + {% for observation in models[model_component]['observations'] %} + GetObservations-{{observation}}-{{model_component}} => RenderJediObservations-{{model_component}} + {% endfor %} RenderJediObservations-{{model_component}} => RunJediFgatExecutable-{{model_component}} # Run analysis diagnostics @@ -199,8 +204,19 @@ [[StageJediCycle-{{model_component}}]] script = "swell task StageJedi $config -d $datetime -m {{model_component}}" - [[GetObservations-{{model_component}}]] - script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + # --------------------------------------------------------------------------------------------- + # This has to be here so swell will successfully register the task + # [[GetObservations-{{model_component}}]] + # script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + # --------------------------------------------------------------------------------------------- + + {% for observation in models[model_component]['observations'] %} + [[GetObservations-{{observation}}-{{model_component}}]] + script = "swell task GetObservations $config -d $datetime -m {{model_component}} -a {{observation}}" + [[[environment]]] + OBSERVATION = {{observation}} + {% endfor %} + # --------------------------------------------------------------------------------------------- [[GenerateBClimatology-{{model_component}}]] script = "swell task GenerateBClimatology $config -d $datetime -m {{model_component}}" diff --git a/src/swell/suites/localensembleda/flow.cylc b/src/swell/suites/localensembleda/flow.cylc index 82e351c6d..ac17c41c6 100644 --- a/src/swell/suites/localensembleda/flow.cylc +++ b/src/swell/suites/localensembleda/flow.cylc @@ -192,7 +192,7 @@ {% for observation in models[model_component]['observations'] %} [[GetObservations-{{observation}}-{{model_component}}]] - script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + script = "swell task GetObservations $config -d $datetime -m {{model_component}} -a {{observation}}" [[[environment]]] OBSERVATION = {{observation}} {% endfor %} From 20ccbcf0da6a9260861209c9feaa6e143733e872 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 5 Mar 2026 16:11:56 -0500 Subject: [PATCH 10/18] add key for concurrent limit --- src/swell/suites/3dfgat_cycle/flow.cylc | 2 +- src/swell/suites/3dvar/flow.cylc | 2 +- src/swell/suites/3dvar_atmos/flow.cylc | 2 +- src/swell/suites/3dvar_cycle/flow.cylc | 2 +- src/swell/suites/hofx/flow.cylc | 2 +- src/swell/suites/hofx_cf/flow.cylc | 2 +- src/swell/suites/localensembleda/flow.cylc | 2 +- src/swell/suites/suite_questions.py | 1 + src/swell/utilities/question_defaults.py | 10 ++++++++++ 9 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/swell/suites/3dfgat_cycle/flow.cylc b/src/swell/suites/3dfgat_cycle/flow.cylc index c4e82a4be..630ceb34d 100644 --- a/src/swell/suites/3dfgat_cycle/flow.cylc +++ b/src/swell/suites/3dfgat_cycle/flow.cylc @@ -23,7 +23,7 @@ runahead limit = {{runahead_limit}} [[queues]] [[[default]]] - limit=6 + limit = {{concurrent_limit}} [[graph]] R1 = """ diff --git a/src/swell/suites/3dvar/flow.cylc b/src/swell/suites/3dvar/flow.cylc index 67309d0dc..39cbb92b5 100644 --- a/src/swell/suites/3dvar/flow.cylc +++ b/src/swell/suites/3dvar/flow.cylc @@ -23,7 +23,7 @@ runahead limit = {{runahead_limit}} [[queues]] [[[default]]] - limit=6 + limit = {{concurrent_limit}} [[graph]] R1 = """ diff --git a/src/swell/suites/3dvar_atmos/flow.cylc b/src/swell/suites/3dvar_atmos/flow.cylc index b648c7ea1..a86eea2c7 100644 --- a/src/swell/suites/3dvar_atmos/flow.cylc +++ b/src/swell/suites/3dvar_atmos/flow.cylc @@ -23,7 +23,7 @@ runahead limit = {{runahead_limit}} [[queues]] [[[default]]] - limit=6 + limit = {{concurrent_limit}} [[graph]] R1 = """ diff --git a/src/swell/suites/3dvar_cycle/flow.cylc b/src/swell/suites/3dvar_cycle/flow.cylc index a19417415..1521697e3 100644 --- a/src/swell/suites/3dvar_cycle/flow.cylc +++ b/src/swell/suites/3dvar_cycle/flow.cylc @@ -22,7 +22,7 @@ final cycle point = {{final_cycle_point}} [[queues]] [[[default]]] - limit=6 + limit = {{concurrent_limit}} [[graph]] R1 = """ diff --git a/src/swell/suites/hofx/flow.cylc b/src/swell/suites/hofx/flow.cylc index dec8dddc2..7f62faf43 100644 --- a/src/swell/suites/hofx/flow.cylc +++ b/src/swell/suites/hofx/flow.cylc @@ -23,7 +23,7 @@ runahead limit = {{runahead_limit}} [[queues]] [[[default]]] - limit=6 + limit = {{concurrent_limit}} [[graph]] R1 = """ diff --git a/src/swell/suites/hofx_cf/flow.cylc b/src/swell/suites/hofx_cf/flow.cylc index 5695cd301..b1497f8e8 100644 --- a/src/swell/suites/hofx_cf/flow.cylc +++ b/src/swell/suites/hofx_cf/flow.cylc @@ -23,7 +23,7 @@ runahead limit = {{runahead_limit}} [[queues]] [[[default]]] - limit=6 + limit = {{concurrent_limit}} [[graph]] R1 = """ diff --git a/src/swell/suites/localensembleda/flow.cylc b/src/swell/suites/localensembleda/flow.cylc index ac17c41c6..e25e7136d 100644 --- a/src/swell/suites/localensembleda/flow.cylc +++ b/src/swell/suites/localensembleda/flow.cylc @@ -23,7 +23,7 @@ runahead limit = {{runahead_limit}} [[queues]] [[[default]]] - limit=6 + limit = {{concurrent_limit}} [[graph]] R1 = """ diff --git a/src/swell/suites/suite_questions.py b/src/swell/suites/suite_questions.py index 247bee869..c9cf34ba1 100644 --- a/src/swell/suites/suite_questions.py +++ b/src/swell/suites/suite_questions.py @@ -41,6 +41,7 @@ class SuiteQuestions(QuestionContainer, Enum): qd.final_cycle_point(), qd.model_components(), qd.runahead_limit() + qd.concurrent_limit() ] ) diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index bf11de750..824340757 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -34,6 +34,16 @@ class comparison_experiment_paths(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass + class concurrent_limit(SuiteQuestion): + default_value: int = 10 + question_name: str = "concurrent_limit" + ask_question: bool = False + prompt: str = "Cylc parameter for how many jobs can run concurrently." + widget_type: WType = WType.INTEGER + + # -------------------------------------------------------------------------------------------------- + @dataclass class cycle_times(SuiteQuestion): default_value: str = "defer_to_model" From 7a670a91e6bc48eefb96b140baa2479dd775c203 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 5 Mar 2026 16:20:41 -0500 Subject: [PATCH 11/18] fixes --- src/swell/suites/suite_questions.py | 2 +- src/swell/swell.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/swell/suites/suite_questions.py b/src/swell/suites/suite_questions.py index c9cf34ba1..6e695730b 100644 --- a/src/swell/suites/suite_questions.py +++ b/src/swell/suites/suite_questions.py @@ -40,7 +40,7 @@ class SuiteQuestions(QuestionContainer, Enum): qd.start_cycle_point(), qd.final_cycle_point(), qd.model_components(), - qd.runahead_limit() + qd.runahead_limit(), qd.concurrent_limit() ] ) diff --git a/src/swell/swell.py b/src/swell/swell.py index 027aeee14..9ee25e075 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -187,7 +187,8 @@ def launch( @click.argument('config') @click.option('-d', '--datetime', 'datetime', default=None, help=datetime_help) @click.option('-m', '--model', 'model', default=None, help=model_help) -@click.option('-a', '--additional-parameter', 'additional_parameter', default=None, help=additional_parameter_help) +@click.option('-a', '--additional-parameter', 'additional_parameter', + default=None, help=additional_parameter_help) @click.option('-p', '--ensemblePacket', 'ensemblePacket', default=None, help=ensemble_help) def task( task: str, From bd050fffbadae41882f0e107af47978cf00c4274 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 5 Mar 2026 16:44:06 -0500 Subject: [PATCH 12/18] add 3dfgat_atmos --- src/swell/suites/3dfgat_atmos/flow.cylc | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/swell/suites/3dfgat_atmos/flow.cylc b/src/swell/suites/3dfgat_atmos/flow.cylc index b4d238308..a189cba0c 100644 --- a/src/swell/suites/3dfgat_atmos/flow.cylc +++ b/src/swell/suites/3dfgat_atmos/flow.cylc @@ -21,6 +21,9 @@ initial cycle point = {{start_cycle_point}} final cycle point = {{final_cycle_point}} runahead limit = {{runahead_limit}} + [[queues]] + [[[default]]] + limit = {{concurrent_limit}} [[graph]] R1 = """ @@ -58,13 +61,17 @@ {% if cycling_varbc %} # Cycling VarBC is active, biases from the previous cycle will be used - RunJediVariationalExecutable-{{model_component}}[-PT6H] => GetObservations-{{model_component}} + RunJediVariationalExecutable-{{model_component}}[-PT6H] => GetObservationsStart {% else %} # Cycling VarBC is inactive, static bias files will be used - GetObservations-{{model_component}} + GetObservationsStart {% endif %} + {% for observation in models[model_component]['observations'] %} + GetObservationsStart => GetObservations-{{observation}}-{{model_component}} => RenderJediObservations-{{model_component}} + {% endfor %} + # Perform staging that is cycle dependent StageJediCycle-{{model_component}} @@ -152,6 +159,9 @@ [[GetBackgroundGeosExperiment-{{model_component}} ]] script = "swell task GetBackgroundGeosExperiment $config -d $datetime -m {{model_component}}" + [[GetObservationsStart]] + script = "true" + [[GetObservations-{{model_component}}]] script = "swell task GetObservations $config -d $datetime -m {{model_component}}" From 62279614284800587e91c76f55fb732f51f26f40 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 5 Mar 2026 17:02:17 -0500 Subject: [PATCH 13/18] add hofx_cf --- src/swell/suites/hofx_cf/flow.cylc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/swell/suites/hofx_cf/flow.cylc b/src/swell/suites/hofx_cf/flow.cylc index b1497f8e8..22e2c7f7e 100644 --- a/src/swell/suites/hofx_cf/flow.cylc +++ b/src/swell/suites/hofx_cf/flow.cylc @@ -121,14 +121,12 @@ # --------------------------------------------------------------------------------------------- # This has to be here so swell will successfully register the task # [[GetObservations-{{model_component}}]] - # script = "swell task GetObservations $config -d $datetime -m {{model_component}} -a {{observation}}" + # script = "swell task GetObservations $config -d $datetime -m {{model_component}}" # --------------------------------------------------------------------------------------------- {% for observation in models[model_component]['observations'] %} [[GetObservations-{{observation}}-{{model_component}}]] - script = "swell task GetObservations $config -d $datetime -m {{model_component}}" - [[[environment]]] - OBSERVATION = {{observation}} + script = "swell task GetObservations $config -d $datetime -m {{model_component}} -a {{observation}}" {% endfor %} # --------------------------------------------------------------------------------------------- From afd1febc1a8893d8b028e59935544b3d821fe966 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 6 Mar 2026 10:54:36 -0500 Subject: [PATCH 14/18] Add GetObservations group to hofx_cf --- src/swell/suites/hofx_cf/flow.cylc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/swell/suites/hofx_cf/flow.cylc b/src/swell/suites/hofx_cf/flow.cylc index 22e2c7f7e..29afcbdac 100644 --- a/src/swell/suites/hofx_cf/flow.cylc +++ b/src/swell/suites/hofx_cf/flow.cylc @@ -119,14 +119,16 @@ script = "swell task GetBackground $config -d $datetime -m {{model_component}}" # --------------------------------------------------------------------------------------------- - # This has to be here so swell will successfully register the task - # [[GetObservations-{{model_component}}]] - # script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + + [[GetObservations-{{model_component}}]] + script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + # --------------------------------------------------------------------------------------------- {% for observation in models[model_component]['observations'] %} [[GetObservations-{{observation}}-{{model_component}}]] script = "swell task GetObservations $config -d $datetime -m {{model_component}} -a {{observation}}" + inherit = GetObservations-{{model_component}} {% endfor %} # --------------------------------------------------------------------------------------------- From 04220b498b7cfca349d5ee34f04ad8898455fa6a Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 6 Mar 2026 11:07:34 -0500 Subject: [PATCH 15/18] Group GetObs tasks --- src/swell/suites/3dfgat_atmos/flow.cylc | 23 +++++++++++++++++----- src/swell/suites/3dfgat_cycle/flow.cylc | 10 +++++----- src/swell/suites/3dvar_atmos/flow.cylc | 10 +++++----- src/swell/suites/3dvar_cycle/flow.cylc | 10 +++++----- src/swell/suites/hofx/flow.cylc | 10 +++++----- src/swell/suites/localensembleda/flow.cylc | 10 +++++----- 6 files changed, 43 insertions(+), 30 deletions(-) diff --git a/src/swell/suites/3dfgat_atmos/flow.cylc b/src/swell/suites/3dfgat_atmos/flow.cylc index a189cba0c..85ce51923 100644 --- a/src/swell/suites/3dfgat_atmos/flow.cylc +++ b/src/swell/suites/3dfgat_atmos/flow.cylc @@ -69,7 +69,7 @@ {% endif %} {% for observation in models[model_component]['observations'] %} - GetObservationsStart => GetObservations-{{observation}}-{{model_component}} => RenderJediObservations-{{model_component}} + GetObservationsStart => GetObservations-{{observation}}-{{model_component}} => GetObservationsFinish {% endfor %} # Perform staging that is cycle dependent @@ -80,10 +80,9 @@ CloneJedi[^] => StageJediCycle-{{model_component}} StageJediCycle-{{model_component}} => RunJediVariationalExecutable-{{model_component}} GetBackgroundGeosExperiment-{{model_component}}? | GetBackground-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} - GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GetObsNotInR2d2-{{model_component}}: fail? => GetObservationsStart + GetObsNotInR2d2-{{model_component}}? | GetObservationsFinish => RenderJediObservations-{{model_component}} GenerateObservingSystemRecords-{{model_component}} => RenderJediObservations-{{model_component}} - GetObservations-{{model_component}} => RenderJediObservations-{{model_component}} RenderJediObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} @@ -162,8 +161,22 @@ [[GetObservationsStart]] script = "true" + [[GetObservationsFinish]] + script = "true" + + # --------------------------------------------------------------------------------------------- + [[GetObservations-{{model_component}}]] - script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + + # --------------------------------------------------------------------------------------------- + + {% for observation in models[model_component]['observations'] %} + [[GetObservations-{{observation}}-{{model_component}}]] + script = "swell task GetObservations $config -d $datetime -m {{model_component}} -a {{observation}}" + inherit = GetObservations-{{model_component}} + {% endfor %} + # --------------------------------------------------------------------------------------------- [[RenderJediObservations-{{model_component}}]] script = "swell task RenderJediObservations $config -d $datetime -m {{model_component}}" diff --git a/src/swell/suites/3dfgat_cycle/flow.cylc b/src/swell/suites/3dfgat_cycle/flow.cylc index 630ceb34d..9476d7de5 100644 --- a/src/swell/suites/3dfgat_cycle/flow.cylc +++ b/src/swell/suites/3dfgat_cycle/flow.cylc @@ -205,16 +205,16 @@ script = "swell task StageJedi $config -d $datetime -m {{model_component}}" # --------------------------------------------------------------------------------------------- - # This has to be here so swell will successfully register the task - # [[GetObservations-{{model_component}}]] - # script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + + [[GetObservations-{{model_component}}]] + script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + # --------------------------------------------------------------------------------------------- {% for observation in models[model_component]['observations'] %} [[GetObservations-{{observation}}-{{model_component}}]] script = "swell task GetObservations $config -d $datetime -m {{model_component}} -a {{observation}}" - [[[environment]]] - OBSERVATION = {{observation}} + inherit = GetObservations-{{model_component}} {% endfor %} # --------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_atmos/flow.cylc b/src/swell/suites/3dvar_atmos/flow.cylc index a86eea2c7..28d5b1758 100644 --- a/src/swell/suites/3dvar_atmos/flow.cylc +++ b/src/swell/suites/3dvar_atmos/flow.cylc @@ -162,16 +162,16 @@ script = "swell task GetBackgroundGeosExperiment $config -d $datetime -m {{model_component}}" # --------------------------------------------------------------------------------------------- - # This has to be here so swell will successfully register the task - # [[GetObservations-{{model_component}}]] - # script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + + [[GetObservations-{{model_component}}]] + script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + # --------------------------------------------------------------------------------------------- {% for observation in models[model_component]['observations'] %} [[GetObservations-{{observation}}-{{model_component}}]] script = "swell task GetObservations $config -d $datetime -m {{model_component}} -a {{observation}}" - [[[environment]]] - OBSERVATION = {{observation}} + inherit = GetObservations-{{model_component}} {% endfor %} # --------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_cycle/flow.cylc b/src/swell/suites/3dvar_cycle/flow.cylc index 1521697e3..ab5807473 100644 --- a/src/swell/suites/3dvar_cycle/flow.cylc +++ b/src/swell/suites/3dvar_cycle/flow.cylc @@ -204,16 +204,16 @@ script = "swell task StageJedi $config -d $datetime -m {{model_component}}" # --------------------------------------------------------------------------------------------- - # This has to be here so swell will successfully register the task - # [[GetObservations-{{model_component}}]] - # script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + + [[GetObservations-{{model_component}}]] + script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + # --------------------------------------------------------------------------------------------- {% for observation in models[model_component]['observations'] %} [[GetObservations-{{observation}}-{{model_component}}]] script = "swell task GetObservations $config -d $datetime -m {{model_component}} -a {{observation}}" - [[[environment]]] - OBSERVATION = {{observation}} + inherit = GetObservations-{{model_component}} {% endfor %} # --------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/hofx/flow.cylc b/src/swell/suites/hofx/flow.cylc index 7f62faf43..bcb3bc31c 100644 --- a/src/swell/suites/hofx/flow.cylc +++ b/src/swell/suites/hofx/flow.cylc @@ -144,16 +144,16 @@ script = "swell task GetBackgroundGeosExperiment $config -d $datetime -m {{model_component}}" # --------------------------------------------------------------------------------------------- - # This has to be here so swell will successfully register the task - # [[GetObservations-{{model_component}}]] - # script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + + [[GetObservations-{{model_component}}]] + script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + # --------------------------------------------------------------------------------------------- {% for observation in models[model_component]['observations'] %} [[GetObservations-{{observation}}-{{model_component}}]] script = "swell task GetObservations $config -d $datetime -m {{model_component}} -a {{observation}}" - [[[environment]]] - OBSERVATION = {{observation}} + inherit = GetObservations-{{model_component}} {% endfor %} # --------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/localensembleda/flow.cylc b/src/swell/suites/localensembleda/flow.cylc index e25e7136d..144b636b9 100644 --- a/src/swell/suites/localensembleda/flow.cylc +++ b/src/swell/suites/localensembleda/flow.cylc @@ -185,16 +185,16 @@ {%- endfor %} # --------------------------------------------------------------------------------------------- - # This has to be here so swell will successfully register the task - # [[GetObservations-{{model_component}}]] - # script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + + [[GetObservations-{{model_component}}]] + script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + # --------------------------------------------------------------------------------------------- {% for observation in models[model_component]['observations'] %} [[GetObservations-{{observation}}-{{model_component}}]] script = "swell task GetObservations $config -d $datetime -m {{model_component}} -a {{observation}}" - [[[environment]]] - OBSERVATION = {{observation}} + inherit = GetObservations-{{model_component}} {% endfor %} # --------------------------------------------------------------------------------------------- From d99a8c69600cd3af3de2cf2ccc944ddff6bab1ad Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 12 Mar 2026 16:57:17 -0400 Subject: [PATCH 16/18] Fix GetObs group --- src/swell/suites/3dvar/flow.cylc | 7 +++---- src/swell/suites/3dvar_atmos/flow.cylc | 2 -- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/swell/suites/3dvar/flow.cylc b/src/swell/suites/3dvar/flow.cylc index 39cbb92b5..803a7a167 100644 --- a/src/swell/suites/3dvar/flow.cylc +++ b/src/swell/suites/3dvar/flow.cylc @@ -131,10 +131,9 @@ script = "swell task GetBackground $config -d $datetime -m {{model_component}}" # --------------------------------------------------------------------------------------------- - # This has to be here so swell will successfully register the task - # [[GetObservations-{{model_component}}]] - # script = "swell task GetObservations $config -d $datetime -m {{model_component}}" - # --------------------------------------------------------------------------------------------- + + [[GetObservations-{{model_component}}]] + script = "swell task GetObservations $config -d $datetime -m {{model_component}}" {% for observation in models[model_component]['observations'] %} [[GetObservations-{{observation}}-{{model_component}}]] diff --git a/src/swell/suites/3dvar_atmos/flow.cylc b/src/swell/suites/3dvar_atmos/flow.cylc index 28d5b1758..8d8c81a3a 100644 --- a/src/swell/suites/3dvar_atmos/flow.cylc +++ b/src/swell/suites/3dvar_atmos/flow.cylc @@ -166,8 +166,6 @@ [[GetObservations-{{model_component}}]] script = "swell task GetObservations $config -d $datetime -m {{model_component}}" - # --------------------------------------------------------------------------------------------- - {% for observation in models[model_component]['observations'] %} [[GetObservations-{{observation}}-{{model_component}}]] script = "swell task GetObservations $config -d $datetime -m {{model_component}} -a {{observation}}" From 17e3345ce4a4a784e8f4670da93513d14aa596f7 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 12 Mar 2026 16:59:38 -0400 Subject: [PATCH 17/18] Group in 3dvar --- src/swell/suites/3dvar/flow.cylc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/swell/suites/3dvar/flow.cylc b/src/swell/suites/3dvar/flow.cylc index 803a7a167..a9a4d7a3f 100644 --- a/src/swell/suites/3dvar/flow.cylc +++ b/src/swell/suites/3dvar/flow.cylc @@ -138,6 +138,7 @@ {% for observation in models[model_component]['observations'] %} [[GetObservations-{{observation}}-{{model_component}}]] script = "swell task GetObservations $config -d $datetime -m {{model_component}} -a {{observation}}" + inherit = GetObservations-{{model_component}} {% endfor %} # --------------------------------------------------------------------------------------------- From 93ad28df1f509795a4f5594821d40d822061120b Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 12 Mar 2026 17:11:53 -0400 Subject: [PATCH 18/18] Add comma --- src/swell/suites/suite_questions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/swell/suites/suite_questions.py b/src/swell/suites/suite_questions.py index ef8c4efdb..ad93fbd5e 100644 --- a/src/swell/suites/suite_questions.py +++ b/src/swell/suites/suite_questions.py @@ -41,8 +41,8 @@ class SuiteQuestions(QuestionContainer, Enum): qd.final_cycle_point(), qd.model_components(), qd.runahead_limit(), - qd.concurrent_limit() - qd.r2d2_experiment_id(), + qd.concurrent_limit(), + qd.r2d2_experiment_id() ] )