diff --git a/src/swell/deployment/platforms/nccs_discover_cascade/task_questions.yaml b/src/swell/deployment/platforms/nccs_discover_cascade/task_questions.yaml index 2f61e0625..c49d2e1a4 100644 --- a/src/swell/deployment/platforms/nccs_discover_cascade/task_questions.yaml +++ b/src/swell/deployment/platforms/nccs_discover_cascade/task_questions.yaml @@ -33,3 +33,6 @@ swell_static_files: ioda_locations_not_in_r2d2: default_value: /discover/nobackup/projects/gmao/dadev/rtodling/archive/542/prePP/ioda + +scratch_root: + default_value: /discover/nobackup/projects/gmao/advda/TSE_staging diff --git a/src/swell/deployment/platforms/nccs_discover_sles15/task_questions.yaml b/src/swell/deployment/platforms/nccs_discover_sles15/task_questions.yaml index 3f0ddf628..5286fc94a 100644 --- a/src/swell/deployment/platforms/nccs_discover_sles15/task_questions.yaml +++ b/src/swell/deployment/platforms/nccs_discover_sles15/task_questions.yaml @@ -39,3 +39,6 @@ swell_static_files: ioda_locations_not_in_r2d2: default_value: /discover/nobackup/projects/gmao/dadev/rtodling/archive/542/prePP/ioda + +scratch_root: + default_value: /discover/nobackup/projects/gmao/advda/TSE_staging diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index 7b6d756bd..daad9ea47 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -11,6 +11,7 @@ import copy import os import yaml +from random import randint from collections.abc import Mapping from typing import Union, Tuple, Optional @@ -334,6 +335,16 @@ def override_with_defaults(self) -> None: if key == 'experiment_id' and val['default_value'] == 'defer_to_code': val['default_value'] = f'swell-{self.suite}' + if key == 'scratch_id' and val['default_value'] == 'defer_to_code': + experiment_id = self.question_dictionary_model_ind['experiment_id']['default_value'] + if experiment_id == 'defer_to_code': + experiment_id = f'swell-{self.suite}' + self.question_dictionary_model_ind['experiment_id'][ + 'default_value'] = experiment_id + + scratch_id = f'{experiment_id}-{randint(0, 99999999):08d}' + val['default_value'] = scratch_id + # ---------------------------------------------------------------------------------------------- def override_with_external(self) -> None: diff --git a/src/swell/suites/3dfgat_atmos/flow.cylc b/src/swell/suites/3dfgat_atmos/flow.cylc index 984445c17..0a873ab28 100644 --- a/src/swell/suites/3dfgat_atmos/flow.cylc +++ b/src/swell/suites/3dfgat_atmos/flow.cylc @@ -98,6 +98,12 @@ """ {% endfor %} + {% for model_component in model_components %} + R1/$ = """ + CleanCycle-{{model_component}} => CleanScratch + """ + {% endfor %} + # -------------------------------------------------------------------------------------------------- [runtime] @@ -185,4 +191,7 @@ script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" {% endfor %} + [[CleanScratch]] + script = "swell task CleanScratch $config" + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dfgat_cycle/flow.cylc b/src/swell/suites/3dfgat_cycle/flow.cylc index 0991725ea..018fe3d68 100644 --- a/src/swell/suites/3dfgat_cycle/flow.cylc +++ b/src/swell/suites/3dfgat_cycle/flow.cylc @@ -119,6 +119,13 @@ {% endfor %} """ {% endfor %} + + {% for model_component in model_components %} + R1/$ = """ + CleanCycle-{{model_component}} => CleanScratch + """ + {% endfor %} + # -------------------------------------------------------------------------------------------------- [runtime] @@ -260,4 +267,7 @@ script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" {% endfor %} + [[CleanScratch]] + script = "swell task CleanScratch $config" + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar/flow.cylc b/src/swell/suites/3dvar/flow.cylc index e25d9244e..3a4a4d4f1 100644 --- a/src/swell/suites/3dvar/flow.cylc +++ b/src/swell/suites/3dvar/flow.cylc @@ -82,6 +82,12 @@ """ {% endfor %} + {% for model_component in model_components %} + R1/$ = """ + CleanCycle-{{model_component}} => CleanScratch + """ + {% endfor %} + # -------------------------------------------------------------------------------------------------- [runtime] @@ -166,4 +172,7 @@ script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" {% endfor %} + [[CleanScratch]] + script = "swell task CleanScratch $config" + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_atmos/flow.cylc b/src/swell/suites/3dvar_atmos/flow.cylc index 19ffe8892..50304f1cf 100644 --- a/src/swell/suites/3dvar_atmos/flow.cylc +++ b/src/swell/suites/3dvar_atmos/flow.cylc @@ -97,6 +97,12 @@ """ {% endfor %} + {% for model_component in model_components %} + R1/$ = """ + CleanCycle-{{model_component}} => CleanScratch + """ + {% endfor %} + # -------------------------------------------------------------------------------------------------- [runtime] @@ -184,4 +190,7 @@ script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" {% endfor %} + [[CleanScratch]] + script = "swell task CleanScratch $config" + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_cycle/flow.cylc b/src/swell/suites/3dvar_cycle/flow.cylc index 16f9b973a..61e94cf09 100644 --- a/src/swell/suites/3dvar_cycle/flow.cylc +++ b/src/swell/suites/3dvar_cycle/flow.cylc @@ -118,6 +118,12 @@ {% endfor %} """ {% endfor %} + {% for model_component in model_components %} + R1/$ = """ + CleanCycle-{{model_component}} => CleanScratch + """ + {% endfor %} + # -------------------------------------------------------------------------------------------------- [runtime] @@ -262,4 +268,7 @@ script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" {% endfor %} + [[CleanScratch]] + script = "swell task CleanScratch $config" + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_geos/flow.cylc b/src/swell/suites/build_geos/flow.cylc index dce2ce3d2..c297ba090 100644 --- a/src/swell/suites/build_geos/flow.cylc +++ b/src/swell/suites/build_geos/flow.cylc @@ -21,7 +21,7 @@ R1 = """ CloneGeos => BuildGeosByLinking? - BuildGeosByLinking:fail? => BuildGeos + BuildGeosByLinking:fail? => BuildGeos => CleanScratch """ # -------------------------------------------------------------------------------------------------- @@ -53,4 +53,7 @@ --{{key}} = {{value}} {%- endfor %} + [[CleanScratch]] + script = "swell task CleanScratch $config" + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_jedi/flow.cylc b/src/swell/suites/build_jedi/flow.cylc index b7e347dee..e7efc65be 100644 --- a/src/swell/suites/build_jedi/flow.cylc +++ b/src/swell/suites/build_jedi/flow.cylc @@ -21,7 +21,7 @@ R1 = """ CloneJedi => BuildJediByLinking? - BuildJediByLinking:fail? => BuildJedi + BuildJediByLinking:fail? => BuildJedi => CleanScratch """ # -------------------------------------------------------------------------------------------------- @@ -53,4 +53,7 @@ --{{key}} = {{value}} {%- endfor %} + [[CleanScratch]] + script = "swell task CleanScratch $config" + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_ncdiags/flow.cylc b/src/swell/suites/convert_ncdiags/flow.cylc index c43986dcd..3737ecb2c 100644 --- a/src/swell/suites/convert_ncdiags/flow.cylc +++ b/src/swell/suites/convert_ncdiags/flow.cylc @@ -54,6 +54,12 @@ """ {% endfor %} + {% for model_component in model_components %} + R1/$ = """ + CleanCycle-{{model_component}} => CleanScratch + """ + {% endfor %} + # -------------------------------------------------------------------------------------------------- [runtime] @@ -99,4 +105,7 @@ [[CleanCycle]] script = "swell task CleanCycle $config -d $datetime -m geos_atmosphere" + [[CleanScratch]] + script = "swell task CleanScratch $config" + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/forecast_geos/flow.cylc b/src/swell/suites/forecast_geos/flow.cylc index cc8d74dea..9e93b839c 100644 --- a/src/swell/suites/forecast_geos/flow.cylc +++ b/src/swell/suites/forecast_geos/flow.cylc @@ -59,6 +59,12 @@ """ {% endfor %} + {% for model_component in model_components %} + R1/$ = """ + CleanCycle-{{model_component}} => CleanScratch + """ + {% endfor %} + # -------------------------------------------------------------------------------------------------- [runtime] @@ -113,4 +119,7 @@ --{{key}} = {{value}} {%- endfor %} + [[CleanScratch]] + script = "swell task CleanScratch $config" + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/geosadas/flow.cylc b/src/swell/suites/geosadas/flow.cylc index f96b3d58e..9a41e189a 100644 --- a/src/swell/suites/geosadas/flow.cylc +++ b/src/swell/suites/geosadas/flow.cylc @@ -58,7 +58,7 @@ GetGeosAdasBackground => RunJediVariationalExecutable # Clean cycle - RunJediVariationalExecutable => CleanCycle + RunJediVariationalExecutable => CleanCycle => CleanScratch """ # -------------------------------------------------------------------------------------------------- @@ -118,4 +118,7 @@ [[CleanCycle]] script = "swell task CleanCycle $config -d $datetime -m geos_atmosphere" + [[CleanScratch]] + script = "swell task CleanScratch $config" + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/hofx/flow.cylc b/src/swell/suites/hofx/flow.cylc index af199788d..1ff463063 100644 --- a/src/swell/suites/hofx/flow.cylc +++ b/src/swell/suites/hofx/flow.cylc @@ -83,6 +83,12 @@ """ {% endfor %} + {% for model_component in model_components %} + R1/$ = """ + CleanCycle-{{model_component}} => CleanScratch + """ + {% endfor %} + # -------------------------------------------------------------------------------------------------- [runtime] @@ -161,4 +167,7 @@ script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" {% endfor %} + [[CleanScratch]] + script = "swell task CleanScratch $config" + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/localensembleda/flow.cylc b/src/swell/suites/localensembleda/flow.cylc index 8126c1d9a..712ed828b 100644 --- a/src/swell/suites/localensembleda/flow.cylc +++ b/src/swell/suites/localensembleda/flow.cylc @@ -93,8 +93,8 @@ # RunJediLocalEnsembleDaExecutable-{{model_component}} => SaveObsDiags-{{model_component}} # Clean up large files - # EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} & - EvaIncrement-{{model_component}} => CleanCycle-{{model_component}} + # EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => + EvaIncrement-{{model_component}} => CleanCycle-{{model_component}} => CleanScratch {% endif %} {% endfor %} @@ -230,4 +230,8 @@ [[sync_point]] script = true + + [[CleanScratch]] + script = "swell task CleanScratch $config" + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/suite_questions.py b/src/swell/suites/suite_questions.py index 247bee869..9aea065b5 100644 --- a/src/swell/suites/suite_questions.py +++ b/src/swell/suites/suite_questions.py @@ -40,7 +40,10 @@ class SuiteQuestions(QuestionContainer, Enum): qd.start_cycle_point(), qd.final_cycle_point(), qd.model_components(), - qd.runahead_limit() + qd.runahead_limit(), + qd.run_in_scratch(), + qd.scratch_root(), + qd.scratch_id(), ] ) diff --git a/src/swell/suites/ufo_testing/flow.cylc b/src/swell/suites/ufo_testing/flow.cylc index 66e8fb888..36bc465a1 100644 --- a/src/swell/suites/ufo_testing/flow.cylc +++ b/src/swell/suites/ufo_testing/flow.cylc @@ -72,6 +72,12 @@ """ {% endfor %} + {% for model_component in model_components %} + R1/$ = """ + CleanCycle-{{model_component}} => CleanScratch + """ + {% endfor %} + # -------------------------------------------------------------------------------------------------- [runtime] @@ -144,4 +150,7 @@ [[CleanCycle]] script = "swell task CleanCycle $config -d $datetime -m geos_atmosphere" + [[CleanScratch]] + script = "swell task CleanScratch $config" + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index 1fa9c7f84..bd8fa623c 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -81,6 +81,12 @@ def __init__( self.__platform__ = self.config.__platform__ self.__suite_to_run__ = self.config.__suite_to_run__ + # Scratch directory information + # ----------------------------- + self.__run_in_scratch__ = self.config.__run_in_scratch__(False) + self.__scratch_root__ = self.config.__scratch_root__(None) + self.__scratch_id__ = self.config.__scratch_id__(None) + if datetime_input is not None: self.__start_cycle_point__ = Datetime(self.config.__start_cycle_point__) self.__final_cycle_point__ = Datetime(self.config.__final_cycle_point__) @@ -144,8 +150,15 @@ def experiment_id(self) -> str: # ---------------------------------------------------------------------------------------------- # Method to get the experiment directory - def experiment_path(self) -> str: - return os.path.join(self.__experiment_root__, self.__experiment_id__) + def experiment_path(self, scratch: bool = True) -> str: + if self.__run_in_scratch__ and scratch: + print(self.__scratch_root__) + print(self.__scratch_id__) + experiment_path = os.path.join(self.__scratch_root__, self.__scratch_id__) + else: + experiment_path = os.path.join(self.__experiment_root__, self.__experiment_id__) + + return experiment_path # ---------------------------------------------------------------------------------------------- @@ -157,11 +170,17 @@ def platform(self) -> str: # Method to get the experiment configuration directory def experiment_config_path(self) -> str: - swell_exp_path = self.experiment_path() + swell_exp_path = self.experiment_path(scratch=False) return os.path.join(swell_exp_path, 'configuration') # ---------------------------------------------------------------------------------------------- + def eva_config_path(self) -> str: + return os.path.join(self.experiment_path(scratch=False), + self.experiment_id()+'-suite', 'eva') + + # ---------------------------------------------------------------------------------------------- + def get_ensemble_packet(self) -> Optional[str]: return self.__ensemble_packet__ diff --git a/src/swell/tasks/clean_scratch.py b/src/swell/tasks/clean_scratch.py new file mode 100644 index 000000000..324cdb301 --- /dev/null +++ b/src/swell/tasks/clean_scratch.py @@ -0,0 +1,41 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. + +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +import os +import shutil +import glob + +from swell.tasks.base.task_base import taskBase + +# -------------------------------------------------------------------------------------------------- + + +class CleanScratch(taskBase): + + """Cleans current cycle based on list defined in the configuration file + + Parameters + ---------- + All inputs are extracted from the JEDI experiment file configuration. + See the taskBase constructor for more information. + + """ + + def execute(self) -> None: + + if self.config.run_in_scratch(): + scratch_path = self.experiment_path(scratch=True) + experiment_path = self.experiment_path(scratch=False) + + files = glob.glob(os.path.join(scratch_path, '*')) + + for file in files: + shutil.copy(file, experiment_path) + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/eva_increment.py b/src/swell/tasks/eva_increment.py index ea95837cc..126130677 100644 --- a/src/swell/tasks/eva_increment.py +++ b/src/swell/tasks/eva_increment.py @@ -33,7 +33,7 @@ def execute(self) -> None: # Read Eva template file into dictionary # -------------------------------------- - eva_path = os.path.join(self.experiment_path(), self.experiment_id()+'-suite', 'eva') + eva_path = self.eva_config_path() eva_config_file = os.path.join(eva_path, f'increment-{model}.yaml') with open(eva_config_file, 'r') as eva_config_file_open: eva_str_template = eva_config_file_open.read() diff --git a/src/swell/tasks/eva_jedi_log.py b/src/swell/tasks/eva_jedi_log.py index 5a73e2558..51de74e3f 100644 --- a/src/swell/tasks/eva_jedi_log.py +++ b/src/swell/tasks/eva_jedi_log.py @@ -30,7 +30,7 @@ def execute(self) -> None: # Read Eva template file into dictionary # -------------------------------------- - eva_path = os.path.join(self.experiment_path(), self.experiment_id()+'-suite', 'eva') + eva_path = self.eva_config_path() eva_config_file = os.path.join(eva_path, f'jedi_log-{model}.yaml') with open(eva_config_file, 'r') as eva_config_file_open: eva_str_template = eva_config_file_open.read() diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index 93353d96b..8c6317c7a 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -64,7 +64,7 @@ def execute(self) -> None: # Read Eva template file into dictionary # -------------------------------------- - eva_path = os.path.join(self.experiment_path(), self.experiment_id()+'-suite', 'eva') + eva_path = self.eva_config_path() eva_config_file = os.path.join(eva_path, f'observations-{model}.yaml') with open(eva_config_file, 'r') as eva_config_file_open: eva_str_template = eva_config_file_open.read() diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index 5d426e941..f65d1457a 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -163,6 +163,36 @@ class runahead_limit(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass + class run_in_scratch(SuiteQuestion): + default_value: bool = False + question_name: str = "run_in_scratch" + ask_question: bool = True + prompt: str = ("Should swell run certain tasks in a specified scratch directory?") + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class scratch_id(SuiteQuestion): + default_value: str = "defer_to_code" + question_name: str = "scratch_id" + ask_question: bool = True + prompt: str = ("Specify an id to place scratch files under the scratch root directory.") + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class scratch_root(SuiteQuestion): + default_value: str = "defer_to_platform" + question_name: str = "scratch_root" + ask_question: bool = True + prompt: str = ("Specify a scratch directory to run tasks in.") + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + @dataclass class skip_ensemble_hofx(SuiteQuestion): default_value: str = "defer_to_model"