Option to execute tasks in scratch area

src/swell/deployment/platforms/nccs_discover_cascade/task_questions.yaml

@@ -33,3 +33,6 @@ swell_static_files:
 
 ioda_locations_not_in_r2d2:
   default_value: /discover/nobackup/projects/gmao/dadev/rtodling/archive/542/prePP/ioda
+
+scratch_root:
+  default_value: /discover/nobackup/projects/gmao/advda/TSE_staging

src/swell/deployment/platforms/nccs_discover_sles15/task_questions.yaml

@@ -33,3 +33,6 @@ swell_static_files:
 
 ioda_locations_not_in_r2d2:
   default_value: /discover/nobackup/projects/gmao/dadev/rtodling/archive/542/prePP/ioda
+
+scratch_root:
+  default_value: /discover/nobackup/projects/gmao/advda/TSE_staging

src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py

@@ -11,6 +11,7 @@
 import copy
 import os
 import yaml
+from random import randint
 from collections.abc import Mapping
 from typing import Union, Tuple, Optional
 
@@ -334,6 +335,16 @@ def override_with_defaults(self) -> None:
             if key == 'experiment_id' and val['default_value'] == 'defer_to_code':
                 val['default_value'] = f'swell-{self.suite}'
 
+            if key == 'scratch_id' and val['default_value'] == 'defer_to_code':
+                experiment_id = self.question_dictionary_model_ind['experiment_id']['default_value']
+                if experiment_id == 'defer_to_code':
+                    experiment_id = f'swell-{self.suite}'
+                    self.question_dictionary_model_ind['experiment_id'][
+                            'default_value'] = experiment_id
+
+                scratch_id = f'{experiment_id}-{randint(0, 99999999):08d}'
+                val['default_value'] = scratch_id
+
     # ----------------------------------------------------------------------------------------------
 
     def override_with_external(self) -> None:

src/swell/suites/3dfgat_atmos/flow.cylc

@@ -98,6 +98,12 @@
         """
         {% endfor %}
 
+        {% for model_component in model_components %}
+        R1/$ = """
+            CleanCycle-{{model_component}} => CleanScratch
+        """
+        {% endfor %}
+
 # --------------------------------------------------------------------------------------------------
 
 [runtime]
@@ -185,4 +191,7 @@
         script = "swell task CleanCycle $config -d $datetime -m {{model_component}}"
     {% endfor %}
 
+    [[CleanScratch]]
+        script = "swell task CleanScratch $config"
+
 # --------------------------------------------------------------------------------------------------

src/swell/suites/3dfgat_cycle/flow.cylc

@@ -120,6 +120,13 @@
         {% endfor %}
         """
         {% endfor %}
+
+        {% for model_component in model_components %}
+        R1/$ = """
+            CleanCycle-{{model_component}} => CleanScratch
+        """
+        {% endfor %}
+
 # --------------------------------------------------------------------------------------------------
 
 [runtime]
@@ -266,4 +273,7 @@
         script = "swell task CleanCycle $config -d $datetime -m {{model_component}}"
     {% endfor %}
 
+    [[CleanScratch]]
+        script = "swell task CleanScratch $config"
+
 # --------------------------------------------------------------------------------------------------

src/swell/suites/3dvar/flow.cylc

@@ -89,6 +89,12 @@
         """
         {% endfor %}
 
+        {% for model_component in model_components %}
+        R1/$ = """
+            CleanCycle-{{model_component}} => CleanScratch
+        """
+        {% endfor %}
+
 # --------------------------------------------------------------------------------------------------
 
 [runtime]
@@ -175,4 +181,7 @@
         script = "swell task CleanCycle $config -d $datetime -m {{model_component}}"
     {% endfor %}
 
+    [[CleanScratch]]
+        script = "swell task CleanScratch $config"
+
 # --------------------------------------------------------------------------------------------------

src/swell/suites/3dvar_atmos/flow.cylc

@@ -97,6 +97,12 @@
         """
         {% endfor %}
 
+        {% for model_component in model_components %}
+        R1/$ = """
+            CleanCycle-{{model_component}} => CleanScratch
+        """
+        {% endfor %}
+
 # --------------------------------------------------------------------------------------------------
 
 [runtime]
@@ -184,4 +190,7 @@
         script = "swell task CleanCycle $config -d $datetime -m {{model_component}}"
     {% endfor %}
 
+    [[CleanScratch]]
+        script = "swell task CleanScratch $config"
+
 # --------------------------------------------------------------------------------------------------

src/swell/suites/3dvar_cycle/flow.cylc

@@ -120,6 +120,12 @@
         {% endfor %}
         """
         {% endfor %}
+        {% for model_component in model_components %}
+        R1/$ = """
+            CleanCycle-{{model_component}} => CleanScratch
+        """
+        {% endfor %}
+
 # --------------------------------------------------------------------------------------------------
 
 [runtime]
@@ -266,4 +272,7 @@
         script = "swell task CleanCycle $config -d $datetime -m {{model_component}}"
     {% endfor %}
 
+    [[CleanScratch]]
+        script = "swell task CleanScratch $config"
+
 # --------------------------------------------------------------------------------------------------

src/swell/suites/build_geos/flow.cylc

@@ -21,7 +21,7 @@
         R1 = """
            CloneGeos => BuildGeosByLinking?
 
-           BuildGeosByLinking:fail? => BuildGeos
+           BuildGeosByLinking:fail? => BuildGeos => CleanScratch
         """
 
 # --------------------------------------------------------------------------------------------------
@@ -53,4 +53,7 @@
             --{{key}} = {{value}}
         {%- endfor %}
 
+    [[CleanScratch]]
+        script = "swell task CleanScratch $config"
+
 # --------------------------------------------------------------------------------------------------

src/swell/suites/build_jedi/flow.cylc

@@ -21,7 +21,7 @@
         R1 = """
            CloneJedi => BuildJediByLinking?
 
-           BuildJediByLinking:fail? => BuildJedi
+           BuildJediByLinking:fail? => BuildJedi => CleanScratch
         """
 
 # --------------------------------------------------------------------------------------------------
@@ -53,4 +53,7 @@
             --{{key}} = {{value}}
         {%- endfor %}
 
+    [[CleanScratch]]
+        script = "swell task CleanScratch $config"
+
 # --------------------------------------------------------------------------------------------------

src/swell/suites/convert_ncdiags/flow.cylc

@@ -54,6 +54,12 @@
         """
         {% endfor %}
 
+        {% for model_component in model_components %}
+        R1/$ = """
+            CleanCycle-{{model_component}} => CleanScratch
+        """
+        {% endfor %}
+
 # --------------------------------------------------------------------------------------------------
 
 [runtime]
@@ -99,4 +105,7 @@
     [[CleanCycle]]
         script = "swell task CleanCycle $config -d $datetime -m geos_atmosphere"
 
+    [[CleanScratch]]
+        script = "swell task CleanScratch $config"
+
 # --------------------------------------------------------------------------------------------------

src/swell/suites/forecast_geos/flow.cylc

@@ -59,6 +59,12 @@
         """
         {% endfor %}
 
+        {% for model_component in model_components %}
+        R1/$ = """
+            CleanCycle-{{model_component}} => CleanScratch
+        """
+        {% endfor %}
+
 # --------------------------------------------------------------------------------------------------
 
 [runtime]
@@ -113,4 +119,7 @@
             --{{key}} = {{value}}
         {%- endfor %}
 
+    [[CleanScratch]]
+        script = "swell task CleanScratch $config"
+
 # --------------------------------------------------------------------------------------------------

src/swell/suites/geosadas/flow.cylc

@@ -58,7 +58,7 @@
             GetGeosAdasBackground => RunJediVariationalExecutable
 
             # Clean cycle
-            RunJediVariationalExecutable => CleanCycle
+            RunJediVariationalExecutable => CleanCycle => CleanScratch
         """
 
 # --------------------------------------------------------------------------------------------------
@@ -118,4 +118,7 @@
     [[CleanCycle]]
         script = "swell task CleanCycle $config -d $datetime -m geos_atmosphere"
 
+    [[CleanScratch]]
+        script = "swell task CleanScratch $config"
+
 # --------------------------------------------------------------------------------------------------

src/swell/suites/hofx/flow.cylc

@@ -83,6 +83,12 @@
         """
         {% endfor %}
 
+        {% for model_component in model_components %}
+        R1/$ = """
+            CleanCycle-{{model_component}} => CleanScratch
+        """
+        {% endfor %}
+
 # --------------------------------------------------------------------------------------------------
 
 [runtime]
@@ -161,4 +167,7 @@
         script = "swell task CleanCycle $config -d $datetime -m {{model_component}}"
     {% endfor %}
 
+    [[CleanScratch]]
+        script = "swell task CleanScratch $config"
+
 # --------------------------------------------------------------------------------------------------

src/swell/suites/localensembleda/flow.cylc

@@ -92,7 +92,7 @@
 
             # Clean up large files
             EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} =>
-            CleanCycle-{{model_component}}
+            CleanCycle-{{model_component}} => CleanScratch
 
         {% endif %}
         {% endfor %}
@@ -225,4 +225,8 @@
 
     [[sync_point]]
         script = true
+
+    [[CleanScratch]]
+        script = "swell task CleanScratch $config"
+
 # --------------------------------------------------------------------------------------------------

src/swell/suites/suite_questions.py

@@ -40,7 +40,10 @@ class SuiteQuestions(QuestionContainer, Enum):
             qd.start_cycle_point(),
             qd.final_cycle_point(),
             qd.model_components(),
-            qd.runahead_limit()
+            qd.runahead_limit(),
+            qd.run_in_scratch(),
+            qd.scratch_root(),
+            qd.scratch_id(),
         ]
     )
 

src/swell/suites/ufo_testing/flow.cylc

@@ -72,6 +72,12 @@
         """
         {% endfor %}
 
+        {% for model_component in model_components %}
+        R1/$ = """
+            CleanCycle-{{model_component}} => CleanScratch
+        """
+        {% endfor %}
+
 # --------------------------------------------------------------------------------------------------
 
 [runtime]
@@ -144,4 +150,7 @@
     [[CleanCycle]]
         script = "swell task CleanCycle $config -d $datetime -m geos_atmosphere"
 
+    [[CleanScratch]]
+        script = "swell task CleanScratch $config"
+
 # --------------------------------------------------------------------------------------------------

src/swell/tasks/base/task_base.py

@@ -81,6 +81,12 @@ def __init__(
         self.__platform__ = self.config.__platform__
         self.__suite_to_run__ = self.config.__suite_to_run__
 
+        # Scratch directory information
+        # -----------------------------
+        self.__run_in_scratch__ = self.config.__run_in_scratch__
+        self.__scratch_root__ = self.config.__scratch_root__
+        self.__scratch_id__ = self.config.__scratch_id__
+
         if datetime_input is not None:
             self.__start_cycle_point__ = Datetime(self.config.__start_cycle_point__)
             self.__final_cycle_point__ = Datetime(self.config.__final_cycle_point__)
@@ -144,8 +150,13 @@ def experiment_id(self) -> str:
     # ----------------------------------------------------------------------------------------------
 
     # Method to get the experiment directory
-    def experiment_path(self) -> str:
-        return os.path.join(self.__experiment_root__, self.__experiment_id__)
+    def experiment_path(self, scratch: bool = True) -> str:
+        if self.__run_in_scratch__ and scratch:
+            experiment_path = os.path.join(self.__scratch_root__, self.__scratch_id__)
+        else:
+            experiment_path = os.path.join(self.__experiment_root__, self.__experiment_id__)
+
+        return experiment_path
 
     # ----------------------------------------------------------------------------------------------
 
@@ -157,11 +168,17 @@ def platform(self) -> str:
 
     # Method to get the experiment configuration directory
     def experiment_config_path(self) -> str:
-        swell_exp_path = self.experiment_path()
+        swell_exp_path = self.experiment_path(scratch=False)
         return os.path.join(swell_exp_path, 'configuration')
 
     # ----------------------------------------------------------------------------------------------
 
+    def eva_config_path(self) -> str:
+        return os.path.join(self.experiment_path(scratch=False),
+                            self.experiment_id()+'-suite', 'eva')
+
+    # ----------------------------------------------------------------------------------------------
+
     def get_ensemble_packet(self) -> Optional[str]:
         return self.__ensemble_packet__
 

src/swell/tasks/clean_scratch.py

@@ -0,0 +1,41 @@
+# (C) Copyright 2021- United States Government as represented by the Administrator of the
+# National Aeronautics and Space Administration. All Rights Reserved.
+
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+
+
+# --------------------------------------------------------------------------------------------------
+
+import os
+from swell.tasks.base.task_base import taskBase
+import glob
+from swell.utilities.shell_commands import run_track_log_subprocess
+
+# --------------------------------------------------------------------------------------------------
+
+
+class CleanScratch(taskBase):
+
+    """Cleans current cycle based on list defined in the configuration file
+
+       Parameters
+       ----------
+         All inputs are extracted from the JEDI experiment file configuration.
+         See the taskBase constructor for more information.
+
+    """
+
+    def execute(self) -> None:
+
+        if self.config.run_in_scratch():
+            scratch_path = self.experiment_path(scratch=True)
+            experiment_path = self.experiment_path(scratch=False)
+
+            files = glob.glob(os.path.join(scratch_path, '*'))
+
+            command = ['cp', '-r'] + files + [experiment_path]
+
+            run_track_log_subprocess(self.logger, command)
+
+# --------------------------------------------------------------------------------------------------