Add suite creation and jedi config render tests

docs/code_tests/code_tests.md

@@ -17,3 +17,11 @@ By default, swell will create several directories in the working directory that
 ```yaml
 test_cache_location: /discover/nobackup/<user>/swell-test-cache
 ```
+
+## Code tests
+
+### Suite creation test
+The suite creation test attempts to construct experiments for all suites within swell in a temporary directory. If one fails, try creating the suite on its own to make sure it is configured properly. Ensure all values are valid and are not filled by the templates `defer_to_model` or `defer_to_platform`.
+
+### JEDI Config test
+The JEDI config test generates mock configs for jedi executables in a dry-run mode, where obs will not be checked and placeholders will be used for experiment filepaths. These configs are compared against reference files located in `src/swell/test/jedi_configs/`, and named `jedi_<suite>_config.yaml`. Any difference in values in these yamls will cause this test to fail, so ensure any differences created are intentional, then run `swell utility CreateMockConfigs` to automatically generate new reference files for all suites. These new files are placed in the `jedi_config` location in the source code.

pyproject.toml

@@ -45,7 +45,8 @@ dependencies = [
     "h5py>=3.7.0",
     "flake8==6.1.0",
     "netCDF4",
-    "ruamel.yaml==0.17.16"
+    "ruamel.yaml==0.17.16",
+    "xarray==2024.7.0"
 ]
 
 [project.optional-dependencies]
@@ -58,7 +59,7 @@ github = [
     "isodate==0.6.1",
     "f90nml==1.4.4",
     "questionary==1.10.0",
-    "netCDF4==1.6.5",
+    "netCDF4==1.6.5"
 ]
 
 [project.urls]
@@ -74,6 +75,7 @@ swell = [
     'deployment/platforms/*/*.yaml',
     'suites/**',
     'test/suite_tests/*.yaml',
+    'test/jedi_configs/*.yaml',
     'configuration/**',
     'utilities/pinned_versions/*.yaml'
 ]

src/swell/configuration/jedi/interfaces/geos_cf/task_questions.yaml

@@ -80,6 +80,9 @@ observations:
   - tropomi_s5p_no2_tropo
   - tempo_no2_total
 
+obs_experiment:
+  default_value: None
+
 observing_system_records_path:
   default_value: None
 

src/swell/deployment/create_experiment.py

@@ -233,7 +233,7 @@ def create_experiment_directory(
     suite_config: str,
     method: str,
     platform: str,
-    override: str,
+    override: str | dict,
     advanced: bool,
     slurm: str | None,
     skip_r2d2: bool
@@ -249,7 +249,12 @@ def create_experiment_directory(
 
     # Read override file
     # ------------------
-    override_dict = read_override_file(override)
+    if isinstance(override, str):
+        override_dict = read_override_file(override)
+    elif override is None:
+        override_dict = {}
+    else:
+        override_dict = override
 
     # Specify whether to skip registering and storing in R2D2
     # -------------------------------------------------------

src/swell/deployment/platforms/nccs_discover_sles15/task_questions.yaml

@@ -19,15 +19,15 @@ existing_jedi_source_directory:
 existing_jedi_source_directory_pinned:
   default_value: /discover/nobackup/projects/gmao/advda/jedi_bundles_sles15/current_pinned_jedi_bundle/source/
 
+existing_perllib_path:
+  default_value: /discover/nobackup/projects/gmao/advda/perllib_opt/GMAO_perllib/g1.0.1/
+
 geos_homdir:
   default_value: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/geos/homdirs/coupled_5deg
 
 initial_restarts_method:
   default_value: geos_expdir
 
-gmao_perllib_path:
-  default_value: /discover/nobackup/projects/gmao/advda/perllib_opt/GMAO_perllib/g1.0.1/
-
 path_to_bufr:
   default_value: None
 

src/swell/suites/geosadas/flow.cylc

@@ -107,7 +107,7 @@
     [[ GetGeosAdasBackground ]]
         script = "swell task GetGeosAdasBackground $config -d $datetime -m geos_atmosphere"
 
-    [[RenderJediObservations-{{model_component}}]]
+    [[RenderJediObservations-geos_atmosphere]]
         script = "swell task RenderJediObservations $config -d $datetime -m geos_atmosphere"
 
     [[RunJediVariationalExecutable]]

src/swell/tasks/render_jedi_observations.py

@@ -51,6 +51,11 @@ def execute(self) -> None:
 
         cwd = os.getcwd()
 
+        if self.config.mock_experiment(False):
+            self.jedi_rendering.add_key('cycle_dir', 'cycle_dir')
+            self.jedi_rendering.add_key('experiment_id', 'experiment_id')
+            self.jedi_rendering.add_key('experiment_root', 'experiment_root')
+
         observations = []
 
         # Iterate through list

src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py

@@ -54,6 +54,13 @@ def execute(self) -> None:
         self.jedi_rendering.add_key('analysis_time', analysis_time)
         self.jedi_rendering.add_key('analysis_time_iso', analysis_time_iso)
 
+        # Add placeholder names if mock experiment
+        # ----------------------------------------
+        if self.config.mock_experiment(False):
+            self.jedi_rendering.add_key('experiment_root', 'experiment_root')
+            self.jedi_rendering.add_key('experiment_id', 'experiment_id')
+            self.jedi_rendering.add_key('cycle_dir', 'cycle_dir')
+
         # Geometry
         # --------
         self.jedi_rendering.add_key('total_processors', self.config.total_processors(None))

src/swell/tasks/run_jedi_ensemble_mean_variance.py

@@ -71,6 +71,13 @@ def execute(self) -> None:
         # Ensemble
         self.jedi_rendering.add_key('ensemble_num_members', self.config.ensemble_num_members(None))
 
+        # Add placeholder names if mock experiment
+        # ----------------------------------------
+        if self.config.mock_experiment(False):
+            self.jedi_rendering.add_key('experiment_root', 'experiment_root')
+            self.jedi_rendering.add_key('experiment_id', 'experiment_id')
+            self.jedi_rendering.add_key('cycle_dir', 'cycle_dir')
+
         # Jedi configuration file
         # -----------------------
         jedi_config_file = os.path.join(self.cycle_dir(), f'jedi_{jedi_application}_config.yaml')

src/swell/tasks/run_jedi_fgat_executable.py

@@ -105,6 +105,13 @@ def execute(self) -> None:
         background_frequency = self.config.background_frequency()
         self.jedi_rendering.add_key('background_frequency', background_frequency)
 
+        # Add placeholder names if mock experiment
+        # ----------------------------------------
+        if self.config.mock_experiment(False):
+            self.jedi_rendering.add_key('experiment_root', 'experiment_root')
+            self.jedi_rendering.add_key('experiment_id', 'experiment_id')
+            self.jedi_rendering.add_key('cycle_dir', 'cycle_dir')
+
         # Use GEOS utility to generate states
         # -----------------------------------
         states = self.geos.states_generator(background_frequency, window_length,

src/swell/tasks/run_jedi_hofx_ensemble_executable.py

@@ -107,6 +107,13 @@ def execute(self) -> None:
         self.jedi_rendering.add_key('ensemble_hofx_packets', ensemble_hofx_packets)
         self.jedi_rendering.add_key('packet_ensemble_members', packet_ensemble_members)
 
+        # Add placeholder names if mock experiment
+        # ----------------------------------------
+        if self.config.mock_experiment(False):
+            self.jedi_rendering.add_key('experiment_root', 'experiment_root')
+            self.jedi_rendering.add_key('experiment_id', 'experiment_id')
+            self.jedi_rendering.add_key('cycle_dir', 'cycle_dir')
+
         # Jedi configuration file
         # -----------------------
         jedi_config_file = os.path.join(self.cycle_dir(),

src/swell/tasks/run_jedi_hofx_executable.py

@@ -84,6 +84,13 @@ def execute(self, ensemble_members: Optional[list] = None) -> None:
         self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None))
         self.jedi_rendering.add_key('window_begin', window_begin)
 
+        # Add placeholder names if mock experiment
+        # ----------------------------------------
+        if self.config.mock_experiment(False):
+            self.jedi_rendering.add_key('experiment_root', 'experiment_root')
+            self.jedi_rendering.add_key('experiment_id', 'experiment_id')
+            self.jedi_rendering.add_key('cycle_dir', 'cycle_dir')
+
         # Model
         # -----
         if window_type == '4D':
@@ -157,6 +164,7 @@ def execute(self, ensemble_members: Optional[list] = None) -> None:
                                jedi_config_file, output_log_file)
             else:
                 self.logger.info('YAML generated, now exiting.')
+                return
 
             # If saving the geovals they need to be combined
             # ----------------------------------------------
@@ -262,14 +270,19 @@ def append_gomsaver(
         # Add mem to the filename if it is not None
         mem_str = f'_mem{mem}' if mem is not None else ''
 
+        if not self.config.mock_experiment(False):
+            cycle_dir = self.cycle_dir()
+        else:
+            cycle_dir = 'cycle_dir'
+
         for observer in jedi_config_dict['observations']['observers']:
 
             observation = observer['observation_name']
 
             # Define the GeoVaLs saver dictionary
             gom_saver_dict = {
                 'filter': 'GOMsaver',
-                'filename': os.path.join(self.cycle_dir(),
+                'filename': os.path.join(cycle_dir,
                                          f'{observation}-geovals.{window_begin}{mem_str}.nc4')
             }
 

src/swell/tasks/run_jedi_local_ensemble_da_executable.py

@@ -140,6 +140,13 @@ def execute(self) -> None:
                                     self.config.local_ensemble_use_linear_observer())
         self.jedi_rendering.add_key('skip_ensemble_hofx', self.config.skip_ensemble_hofx())
 
+        # Add placeholder names if mock experiment
+        # ----------------------------------------
+        if self.config.mock_experiment(False):
+            self.jedi_rendering.add_key('experiment_root', 'experiment_root')
+            self.jedi_rendering.add_key('experiment_id', 'experiment_id')
+            self.jedi_rendering.add_key('cycle_dir', 'cycle_dir')
+
         # Prevent both 'local_ensemble_save_posterior_mean' and
         # 'local_ensemble_save_posterior_ensemble' from being true
         # --------------------------------------------------------

src/swell/tasks/run_jedi_obsfilters_executable.py

@@ -76,6 +76,13 @@ def execute(self, ensemble_members: Optional[list] = None) -> None:
         self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None))
         self.jedi_rendering.add_key('window_begin', window_begin)
 
+        # Add placeholder names if mock experiment
+        # ----------------------------------------
+        if self.config.mock_experiment(False):
+            self.jedi_rendering.add_key('experiment_root', 'experiment_root')
+            self.jedi_rendering.add_key('experiment_id', 'experiment_id')
+            self.jedi_rendering.add_key('cycle_dir', 'cycle_dir')
+
         # Model
         # -----
         if window_type == '4D':

src/swell/tasks/run_jedi_ufo_tests_executable.py

@@ -57,6 +57,13 @@ def execute(self) -> None:
         self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None))
         self.jedi_rendering.add_key('window_begin', window_begin)
 
+        # Add placeholder names if mock experiment
+        # ----------------------------------------
+        if self.config.mock_experiment(False):
+            self.jedi_rendering.add_key('experiment_root', 'experiment_root')
+            self.jedi_rendering.add_key('experiment_id', 'experiment_id')
+            self.jedi_rendering.add_key('cycle_dir', 'cycle_dir')
+
         # Open the JEDI config file and fill templates
         # --------------------------------------------
         jedi_config_dict = self.jedi_rendering.render_oops_file(f'{jedi_application}', '3D')

src/swell/tasks/run_jedi_variational_executable.py

@@ -96,6 +96,14 @@ def execute(self) -> None:
         self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None))
         self.jedi_rendering.add_key('window_begin', window_begin)
 
+        # Add placeholder names if mock experiment
+        # ----------------------------------------
+        if self.config.mock_experiment(False):
+            print('MOCK EXPERIMENT')
+            self.jedi_rendering.add_key('experiment_root', 'experiment_root')
+            self.jedi_rendering.add_key('experiment_id', 'experiment_id')
+            self.jedi_rendering.add_key('cycle_dir', 'cycle_dir')
+
         # Atmosphere background error model
         # ---------------------------------
         if gsibec_configuration is not None:

src/swell/tasks/task_questions.py

@@ -572,7 +572,8 @@ class TaskQuestions(QuestionContainer, Enum):
             qd.background_time_offset(),
             qd.observing_system_records_path(),
             qd.observations(),
-            qd.window_length()
+            qd.window_length(),
+            qd.mock_experiment()
         ]
     )
 
@@ -590,6 +591,7 @@ class TaskQuestions(QuestionContainer, Enum):
             qd.window_length(),
             qd.window_type(),
             qd.comparison_log_type('convert_state_soca2cice'),
+            qd.mock_experiment()
         ]
     )
 
@@ -607,6 +609,7 @@ class TaskQuestions(QuestionContainer, Enum):
             qd.observations(),
             qd.observing_system_records_path(),
             qd.comparison_log_type('ensmeanvariance'),
+            qd.mock_experiment()
         ]
     )
 
@@ -617,7 +620,8 @@ class TaskQuestions(QuestionContainer, Enum):
         questions=[
             run_jedi_executable,
             qd.marine_models(),
-            qd.comparison_log_type('fgat')
+            qd.comparison_log_type('fgat'),
+            qd.mock_experiment()
         ]
     )
 
@@ -636,7 +640,8 @@ class TaskQuestions(QuestionContainer, Enum):
             qd.generate_yaml_and_exit(),
             qd.jedi_forecast_model(),
             qd.total_processors(),
-            qd.comparison_log_type('hofx')
+            qd.comparison_log_type('hofx'),
+            qd.mock_experiment()
         ]
     )
 
@@ -654,6 +659,7 @@ class TaskQuestions(QuestionContainer, Enum):
             qd.save_geovals(),
             qd.total_processors(),
             qd.comparison_log_type('ensemblehofx'),
+            qd.mock_experiment()
         ]
     )
 
@@ -694,6 +700,7 @@ class TaskQuestions(QuestionContainer, Enum):
             qd.vertical_localization_method(),
             qd.perhost(),
             qd.comparison_log_type('localensembleda'),
+            qd.mock_experiment()
         ]
     )
 
@@ -711,7 +718,8 @@ class TaskQuestions(QuestionContainer, Enum):
             qd.observing_system_records_path(),
             qd.total_processors(),
             qd.obs_thinning_rej_fraction(),
-            qd.comparison_log_type('obsfilters')
+            qd.comparison_log_type('obsfilters'),
+            qd.mock_experiment()
         ]
     )
 
@@ -725,6 +733,7 @@ class TaskQuestions(QuestionContainer, Enum):
             qd.single_observations(),
             qd.window_length(),
             qd.comparison_log_type('ufo_tests'),
+            qd.mock_experiment()
         ]
     )
 
@@ -736,6 +745,7 @@ class TaskQuestions(QuestionContainer, Enum):
             run_jedi_executable,
             qd.perhost(),
             qd.comparison_log_type('variational'),
+            qd.mock_experiment()
         ]
     )
 

src/swell/test/code_tests/code_tests.py

@@ -17,6 +17,8 @@
 from swell.test.code_tests.unused_variables_test import UnusedVariablesTest
 from swell.test.code_tests.question_dictionary_comparison_test import QuestionDictionaryTest
 from swell.test.code_tests.test_generate_observing_system import GenerateObservingSystemTest
+from swell.test.code_tests.suite_creation_test import SuiteCreationTest
+from swell.test.code_tests.jedi_config_test import JEDIConfigTest
 
 
 # --------------------------------------------------------------------------------------------------
@@ -52,6 +54,12 @@ def code_tests() -> None:
     # Load Pinned Versions Test
     test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(PinnedVersionsTest))
 
+    # Load Suite Creation Test
+    test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(SuiteCreationTest))
+
+    # Load Suite Creation Test
+    test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(JEDIConfigTest))
+
     # Create a test runner
     test_runner = unittest.TextTestRunner()