diff --git a/fre/__init__.py b/fre/__init__.py index cfe401402..c0f5f4a01 100644 --- a/fre/__init__.py +++ b/fre/__init__.py @@ -10,7 +10,7 @@ fre_logger = logging.getLogger(__name__) -FORMAT = "[%(levelname)5s:%(filename)24s:%(funcName)24s] %(message)s" +FORMAT = "[%(levelname)7s:%(filename)25s:%(funcName)24s] %(message)s" logging.basicConfig(level = logging.WARNING, format = FORMAT, filename = None, diff --git a/fre/make/create_checkout_script.py b/fre/make/create_checkout_script.py index 0dedc3ec3..5e18c0e7a 100644 --- a/fre/make/create_checkout_script.py +++ b/fre/make/create_checkout_script.py @@ -24,7 +24,7 @@ def baremetal_checkout_write(model_yaml, src_dir, jobs, pc, execute): # Make checkout script executable os.chmod(src_dir+"/checkout.sh", 0o744) - fre_logger.info("Checkout script created in %s/checkout.sh", src_dir ) + fre_logger.info("Checkout script created: %s/checkout.sh", src_dir ) if execute: fre_checkout.run() @@ -36,7 +36,7 @@ def container_checkout_write(model_yaml, src_dir, tmp_dir, jobs, pc): fre_checkout = checkout.checkoutForContainer("checkout.sh", src_dir, tmp_dir) fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(), jobs, pc) fre_checkout.finish(model_yaml.compile.getCompileYaml(), pc) - fre_logger.info("Checkout script created in ./%s/checkout.sh", tmp_dir) + fre_logger.info("Checkout script created: ./%s/checkout.sh", tmp_dir) def checkout_create(yamlfile: str, platform: str, target: str, no_parallel_checkout: Optional[bool] = None, njobs: int = 4, execute: Optional[bool] = False, force_checkout: Optional[bool] = False): @@ -106,6 +106,7 @@ def checkout_create(yamlfile: str, platform: str, target: str, no_parallel_check target = targetfre.fretarget(target_name) fre_logger.setLevel(level = logging.INFO) +# fre_logger.info("") ## Loop through the platforms specified on the command line ## If the platform is a baremetal platform, write the checkout script and run it once ## This should be done separately and serially because bare metal platforms should all be using @@ -129,8 +130,8 @@ def checkout_create(yamlfile: str, platform: str, target: str, no_parallel_check # Create and run (if --execute passed) the checkout script baremetal_checkout_write(model_yaml, src_dir, jobs, pc, execute) elif os.path.exists(f"{src_dir}/checkout.sh") and force_checkout: - fre_logger.info("Checkout script PREVIOUSLY created in %s/checkout.sh", src_dir) - fre_logger.info("*** REMOVING CHECKOUT SCRIPT ***") + fre_logger.warning("Checkout script PREVIOUSLY created in %s/checkout.sh", src_dir) + fre_logger.warning("*** REMOVING CHECKOUT SCRIPT ***") # Remove the checkout script os.remove(f"{src_dir}/checkout.sh") @@ -138,16 +139,13 @@ def checkout_create(yamlfile: str, platform: str, target: str, no_parallel_check baremetal_checkout_write(model_yaml, src_dir, jobs, pc, execute) elif os.path.exists(f"{src_dir}/checkout.sh") and not force_checkout: - fre_logger.info("Checkout script PREVIOUSLY created in %s/checkout.sh", src_dir) + fre_logger.warning("Checkout script PREVIOUSLY created in %s/checkout.sh", src_dir) if execute: try: subprocess.run(args=[src_dir+"/checkout.sh"], check=True) except Exception as exc: raise OSError(f"\nThere was an error with the checkout script {src_dir}/checkout.sh.", f"\nTry removing test folder: {platform['modelRoot']}\n") from exc - else: - return - else: src_dir = f'{platform["modelRoot"]}/{fremake_yaml["experiment"]}/src' tmp_dir = f"tmp/{platform_name}" @@ -159,7 +157,7 @@ def checkout_create(yamlfile: str, platform: str, target: str, no_parallel_check container_checkout_write(model_yaml, src_dir, tmp_dir, jobs, pc) # If checkout script exists and force_checkout is used: elif os.path.exists(f"{tmp_dir}/checkout.sh") and force_checkout: - fre_logger.info("Checkout script PREVIOUSLY created in %s/checkout.sh", tmp_dir) + fre_logger.info("Checkout script PREVIOUSLY created: %s/checkout.sh", tmp_dir) fre_logger.info("*** REMOVING CHECKOUT SCRIPT ***") # remove os.remove(f"{tmp_dir}/checkout.sh") @@ -167,4 +165,5 @@ def checkout_create(yamlfile: str, platform: str, target: str, no_parallel_check container_checkout_write(model_yaml, src_dir, tmp_dir, jobs, pc) # If checkout script exists but force_checkout is not used elif os.path.exists(f"{tmp_dir}/checkout.sh") and not force_checkout: - fre_logger.info("Checkout script PREVIOUSLY created in %s/checkout.sh", tmp_dir) + fre_logger.info("Checkout script PREVIOUSLY created: %s/checkout.sh", tmp_dir) + fre_logger.info("") diff --git a/fre/make/create_compile_script.py b/fre/make/create_compile_script.py index a874b98bb..de9303723 100644 --- a/fre/make/create_compile_script.py +++ b/fre/make/create_compile_script.py @@ -1,23 +1,55 @@ ''' Creates a compile script to compile the model and generate a model executable. ''' - -import os import logging from pathlib import Path from multiprocessing.dummy import Pool +#import filecmp +#import difflib -import fre.yamltools.combine_yamls_script as cy from typing import Optional +import fre.yamltools.combine_yamls_script as cy from fre.make.make_helpers import get_mktemplate_path from .gfdlfremake import varsfre, yamlfre, targetfre, buildBaremetal fre_logger = logging.getLogger(__name__) +def compile_call(fremake_yaml, template_path, src_dir, bld_dir, target, platform, jobs): + """ + lkjsdlfkjs + + :param fremake_yaml: + :type fremake_yaml: + :param template_path: + :type template_path: + :param src_dir: + :type src_dir: + :param bld_dir: + :type bld_dir: + :param target: + :type target: + :param platform: + :type platform: + :param jobs: + :type jobs: + """ + fremake_build = buildBaremetal.buildBaremetal(exp=fremake_yaml["experiment"], + mkTemplatePath=template_path, + srcDir=src_dir, + bldDir=bld_dir, + target=target, + env_setup=platform["envSetup"], + jobs=jobs) + for c in fremake_yaml['src']: + fremake_build.writeBuildComponents(c) + fremake_build.writeScript() + fre_logger.info("Compile script created: %s/compile.sh", bld_dir) + def compile_create(yamlfile:str, platform:str, target:str, njobs: int = 4, nparallel: int = 1, execute: Optional[bool] = False, - verbose: Optional[bool] = None): + verbose: Optional[bool] = False, + force_compile: Optional[bool] = False): """ Creates the compile script for bare-metal build @@ -28,7 +60,8 @@ def compile_create(yamlfile:str, platform:str, target:str, njobs: int = 4, :type platform: str :param target: Predefined FRE targets; options include [prod/debug/repro]-openmp :type target: str - :param njobs: Used for parallelism with make; number of files to build simultaneously; on a per-build basis (default 4) + :param njobs: Used for parallelism with make; number of files to build simultaneously; + on a per-build basis (default 4) :type njobs: int :param nparallel: Number of concurrent model builds (default 1) :type nparallel: int @@ -36,6 +69,8 @@ def compile_create(yamlfile:str, platform:str, target:str, njobs: int = 4, :type execute: bool :param verbose: Increase verbosity output :type verbose: bool + :param force_compile: Re-create compile script if specified + :type force_compile: bool :raises ValueError: - Error if platform does not exist in platforms yaml configuration - Error if the mkmf template defined in platforms yaml does not exist @@ -44,23 +79,12 @@ def compile_create(yamlfile:str, platform:str, target:str, njobs: int = 4, # Define variables yml = yamlfile name = yamlfile.split(".")[0] - nparallel = nparallel jobs = str(njobs) if verbose: fre_logger.setLevel(level=logging.DEBUG) - else: - fre_logger.setLevel(level=logging.INFO) - - srcDir = "src" - baremetalRun = False # This is needed if there are no bare metal runs - - ## Split and store the platforms and targets in a list - plist = platform - tlist = target - - # Combined compile yaml file - # combined = Path(f"combined-{name}.yaml") +# else: +# fre_logger.setLevel(level=logging.INFO) # Combine model, compile, and platform yamls full_combined = cy.consolidate_yamls(yamlfile=yml, @@ -74,52 +98,88 @@ def compile_create(yamlfile:str, platform:str, target:str, njobs: int = 4, fre_vars = varsfre.frevars(full_combined) ## Open the yaml file, validate the yaml, and parse as fremake_yaml - modelYaml = yamlfre.freyaml(full_combined, fre_vars) - fremakeYaml = modelYaml.getCompileYaml() + model_yaml = yamlfre.freyaml(full_combined, fre_vars) + fremake_yaml = model_yaml.getCompileYaml() + tlist = target ## Error checking the targets - for targetName in tlist: - target = targetfre.fretarget(targetName) + for target_name in tlist: + target = targetfre.fretarget(target_name) - fremakeBuildList = [] + fremake_build_list = [] ## Loop through platforms and targets - for platformName in plist: - for targetName in tlist: - target = targetfre.fretarget(targetName) - if not modelYaml.platforms.hasPlatform(platformName): - raise ValueError(f"{platformName} does not exist in platforms.yaml") - - platform = modelYaml.platforms.getPlatformFromName(platformName) - ## Make the bldDir based on the modelRoot, the platform, and the target - srcDir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/src" + for platform_name in platform: + for target_name in tlist: + target = targetfre.fretarget(target_name) + if not model_yaml.platforms.hasPlatform(platform_name): + raise ValueError(f"{platform_name} does not exist in platforms.yaml") + + platform = model_yaml.platforms.getPlatformFromName(platform_name) + ## Make the bld_dir based on the modelRoot, the platform, and the target + src_dir = f'{platform["modelRoot"]}/{fremake_yaml["experiment"]}/src' ## Check for type of build if platform["container"] is False: - baremetalRun = True - bldDir = f'{platform["modelRoot"]}/{fremakeYaml["experiment"]}/' + \ - f'{platformName}-{target.gettargetName()}/exec' - os.system("mkdir -p " + bldDir) +# baremetalRun = True + bld_dir = f'{platform["modelRoot"]}/{fremake_yaml["experiment"]}/' + \ + f'{platform_name}-{target.gettargetName()}/exec' + Path(bld_dir).mkdir(parents=True, exist_ok=True) template_path = get_mktemplate_path(mk_template = platform["mkTemplate"], model_root = platform["modelRoot"], container_flag = platform["container"]) - ## Create a list of compile scripts to run in parallel - fremakeBuild = buildBaremetal.buildBaremetal(exp=fremakeYaml["experiment"], - mkTemplatePath=template_path, - srcDir=srcDir, - bldDir=bldDir, - target=target, - env_setup=platform["envSetup"], - jobs=jobs) - for c in fremakeYaml['src']: - fremakeBuild.writeBuildComponents(c) - fremakeBuild.writeScript() - fremakeBuildList.append(fremakeBuild) - fre_logger.info("\nCompile script created at " + bldDir + "/compile.sh" + "\n") - + if not Path(f"{bld_dir}/compile.sh").exists(): + ## Create a list of compile scripts to run in parallel + compile_call(fremake_yaml = fremake_yaml, + template_path = template_path, + src_dir = src_dir, + bld_dir = bld_dir, + target = target, + platform = platform, + jobs = jobs) + fremake_build_list.append(f"{bld_dir}/compile.sh") + elif Path(f"{bld_dir}/compile.sh").exists() and force_compile: + # Remove old compile script + fre_logger.warning("Compile script PREVIOUSLY created: %s/compile.sh", bld_dir) + fre_logger.warning("*** REMOVING COMPILE SCRIPT ***") + Path(f"{bld_dir}/compile.sh").unlink() + + # Re-create compile script + compile_call(fremake_yaml = fremake_yaml, + template_path = template_path, + src_dir = src_dir, + bld_dir = bld_dir, + target = target, + platform = platform, + jobs = jobs) + + fremake_build_list.append(f"{bld_dir}/compile.sh") + elif Path(f"{bld_dir}/compile.sh").exists() and not force_compile: + fre_logger.warning("Compile script PREVIOUSLY created: %s/compile.sh", bld_dir) + fremake_build_list.append(f"{bld_dir}/compile.sh") + ###COMPARE THE TWO TO SEE IF IT'S CHANGED###--> filecmp or difflib + ###IF CHANGED, THROW ERROR### + ###SHOULD IT ALSO BE RE-CREATED IF CHECKOUT RE-CREATED?? --> + ### I THINK THIS WILL BE FOR "ALL" SUBTOOL### + + fre_logger.setLevel(level=logging.INFO) + fre_logger.info("Compile scripts available/generated with specified platform-target combination: ") + for i in fremake_build_list: + fre_logger.info(" - %s", i) + fre_logger.setLevel(level=logging.WARNING) + + # Returns the exit status for multiprocessing pool command if execute: - if baremetalRun: - pool = Pool(processes=nparallel) # Create a multiprocessing Pool - pool.map(buildBaremetal.fremake_parallel, fremakeBuildList) # process data_inputs iterable with pool - else: - return +# if baremetalRun: + # Create a multiprocessing Pool + pool = Pool(processes=nparallel) + # process data_inputs iterable with pool + results = pool.map(buildBaremetal.fremake_parallel, fremake_build_list) + + for r in results: + for key,value in r.items(): + if key == 1: + fre_logger.error("ERROR: compile NOT successful") + fre_logger.error("Check the generated log: %s", value) + elif key == 0: + fre_logger.info("Compile successful") diff --git a/fre/make/create_makefile_script.py b/fre/make/create_makefile_script.py index b8bdcbe11..57f3c7025 100644 --- a/fre/make/create_makefile_script.py +++ b/fre/make/create_makefile_script.py @@ -93,7 +93,7 @@ def makefile_create(yamlfile: str, platform: str, target:str): freMakefile.writeMakefile() former_log_level = fre_logger.level fre_logger.setLevel(logging.INFO) - fre_logger.info("Makefile created in " + bldDir + "/Makefile") + fre_logger.info("Makefile created: " + bldDir + "/Makefile") fre_logger.setLevel(former_log_level) else: bldDir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/exec" @@ -115,5 +115,6 @@ def makefile_create(yamlfile: str, platform: str, target:str): freMakefile.writeMakefile() former_log_level = fre_logger.level fre_logger.setLevel(logging.INFO) - fre_logger.info("Makefile created in " + tmpDir + "/Makefile") + fre_logger.info("Makefile created: " + tmpDir + "/Makefile") fre_logger.setLevel(former_log_level) + fre_logger.warning("") diff --git a/fre/make/fremake.py b/fre/make/fremake.py index f2ab1f76b..0ae16187c 100644 --- a/fre/make/fremake.py +++ b/fre/make/fremake.py @@ -77,15 +77,18 @@ def make_cli(): help = "Use this to run the created compilation script.") @click.option("--force-checkout", is_flag = True, - help = "Force checkout in case the source directory exists.") + help = "Re-create the checkout script in case the source directory exists.") +@click.option("--force-compile", + is_flag = True, + help = "Re-create the compile script in case it exists already.") @click.option("-v", "--verbose", is_flag = True, help = VERBOSE_OPT_HELP) -def all(yamlfile, platform, target, nparallel, njobs, no_parallel_checkout, no_format_transfer, execute, verbose, force_checkout): +def all(yamlfile, platform, target, nparallel, njobs, no_parallel_checkout, no_format_transfer, execute, verbose, force_checkout, force_compile): """ - Perform all fre make functions; run checkout and compile scripts to create model executable or container""" run_fremake_script.fremake_run( - yamlfile, platform, target, nparallel, njobs, no_parallel_checkout, no_format_transfer, execute, verbose, force_checkout) + yamlfile, platform, target, nparallel, njobs, no_parallel_checkout, no_format_transfer, execute, verbose, force_checkout, force_compile) @make_cli.command() @click.option("-y", @@ -177,14 +180,17 @@ def makefile(yamlfile, platform, target): is_flag = True, default = False, help = "Use this to run the created checkout script.") +@click.option("--force-compile", + is_flag = True, + help = "Re-create the compile script in case it exists already.") @click.option("-v", "--verbose", is_flag = True, help = VERBOSE_OPT_HELP) -def compile_script(yamlfile, platform, target, njobs, nparallel, execute, verbose): +def compile_script(yamlfile, platform, target, njobs, nparallel, execute, verbose, force_compile): """ - Write the compile script """ create_compile_script.compile_create( - yamlfile, platform, target, njobs, nparallel, execute, verbose) + yamlfile, platform, target, njobs, nparallel, execute, verbose, force_compile) @make_cli.command @click.option("-y", diff --git a/fre/make/gfdlfremake/buildBaremetal.py b/fre/make/gfdlfremake/buildBaremetal.py index 2a32edb7b..fd1762c8b 100644 --- a/fre/make/gfdlfremake/buildBaremetal.py +++ b/fre/make/gfdlfremake/buildBaremetal.py @@ -6,28 +6,67 @@ import subprocess import os +from pathlib import Path +### TODO run as a batch job on the login cluster def fremake_parallel(fremakeBuildList): """ - Brief: Called for parallel execution purposes. Runs the builds. - Param: - - fremakeBuildList : fremakeBuild object list passes by pool.map + Called for parallel execution purposes. Runs the builds. + + :param fremakeBuildList: list of compile scripts to execute + :type fremakeBuildList: ................. """ - fremakeBuildList.run() + bldDir = Path(fremakeBuildList).parent + + # Run compile script + p1 = subprocess.Popen(fremakeBuildList, stdout=subprocess.PIPE,stderr=subprocess.STDOUT) + + # Direct output to log file as well + p2 = subprocess.Popen(["tee",f"{bldDir}/log.compile"], stdin=p1.stdout) + + # Allow process1 to receive SIGPIPE is process2 exits + p1.stdout.close() + p2.communicate() + + # wait for process1 to finish before checking return code + p1.wait() + if p1.returncode != 0: + return {1: f"{bldDir}/log.compile"} + else: + return {0: f"{bldDir}/log.compile"} class buildBaremetal(): """ - Brief: Creates the build script to compile the model - Param: - - self : The buildScript object - - exp : The experiment name - - mkTemplatePath : The template used by mkmf to compile the model - - srcDir : The source directory - - bldDir : The build directory + Class holding routines that will create the build script to compile the model. + + :ivar str exp: The experiment name + :ivar str mkTemplatePath: The template used by mkmf to compile the model + :ivar str srcDir: The source directory + :ivar str bldDir: The build directory + :ivar str target: + :ivar str env_setup: + :ivar str jobs: """ def __init__(self,exp,mkTemplatePath,srcDir,bldDir,target,env_setup,jobs): """ Initialize variables and set-up the compile script. + + :param self: + :type self: + :param exp: + :type exp: + :param mkTemplatePath: + :type mkTemplatePath: + :param srcDir: + :type srcDir: + :param bldDir: + :type bldDir: + :param target: + :type target: + :param env_setup: + :type env_setup: + :param jobs: + :type jobs: """ self.e = exp self.t = target.gettargetName() @@ -57,10 +96,12 @@ def __init__(self,exp,mkTemplatePath,srcDir,bldDir,target,env_setup,jobs): def writeBuildComponents(self, c): """ - Brief: Adds components to the build script - Param: - - self : The build script object - - c : Component from the compile yaml + Adds components to the build script + + :param self: The build script object + :type self: + :param c: Component from the compile yaml + :type c: """ # Shorthand for component comp = c["component"] @@ -111,9 +152,10 @@ def writeBuildComponents(self, c): ##TODO: add targets input def writeScript(self): """ - Brief: Finishes and writes the build script - Param: - - self : The buildScript object + Finishes and writes the build script + + :param self: The buildScript object + :type self: """ self.f.write(f"cd {self.bld}\n") self.f.write(f"{self.make}\n") @@ -121,30 +163,3 @@ def writeScript(self): # Make compile script executable os.chmod(self.bld+"/compile.sh", 0o744) - -## TODO run as a batch job on the login cluster - def run(self): - """ - Brief: Run the build script - Param: - - self : The dockerfile object - """ - command = [self.bld+"/compile.sh"] - - # Run compile script - p1 = subprocess.Popen(command, stdout=subprocess.PIPE,stderr=subprocess.STDOUT) - - # Direct output to log file as well - p2 = subprocess.Popen(["tee",self.bld+"/log.compile"], stdin=p1.stdout) - - # Allow process1 to receive SIGPIPE is process2 exits - p1.stdout.close() - p2.communicate() - - # wait for process1 to finish before checking return code - p1.wait() - if p1.returncode != 0: - print(f"\nThere was an error running {self.bld}/compile.sh") - print(f"Check the log file: {self.bld}/log.compile") - else: - print(f"\nSuccessful run of {self.bld}/compile.sh") diff --git a/fre/make/gfdlfremake/yamlfre.py b/fre/make/gfdlfremake/yamlfre.py index 50413c4d7..5c9654198 100644 --- a/fre/make/gfdlfremake/yamlfre.py +++ b/fre/make/gfdlfremake/yamlfre.py @@ -5,6 +5,9 @@ from jsonschema import validate, ValidationError, SchemaError from . import platformfre +import logging +fre_logger = logging.getLogger(__name__) + def parseCompile(fname,v): """ Brief: Open the yaml file and parse as fremakeYaml @@ -52,22 +55,19 @@ def __init__(self,compileinfo): try: self.yaml["src"] except: - print("You must set a src to specify the sources in modelRoot/"+self.yaml["experiment"]+"\n") - raise + raise ValueError(f"You must set a src to specify the sources in modelRoot/{self.yaml['experiment']}") ## Loop through the src array for c in self.yaml['src']: ## Check for required component name try: c['component'] except: - print("You must set the 'component' name for each src component") - raise + raise ValueError("You must set the 'component' name for each src component") ## Check for required repo url try: c['repo'] except: - print("'repo' is missing from the component "+c['component']+" in "+self.yaml["experiment"]+"\n") - raise + raise ValueError(f"'repo' is missing from the component {c['component']} in {self.yaml['experiment']}") # Check for optional branch. Otherwise set it to blank try: c['branch'] @@ -116,8 +116,7 @@ def getCompileYaml(self): try: self.yaml except: - print ("You must initialize the compile YAML object before you try to get the yaml \n") - raise + raise ValueError("You must initialize the compile YAML object before you try to get the yaml") return self.yaml class freyaml(): @@ -163,7 +162,7 @@ def __init__(self,combinedyaml,v): schema = json.loads(s) validate(instance=self.freyaml,schema=schema) - print("\nCOMBINED YAML VALID") + fre_logger.info(" *** COMBINED YAML VALID ***") def getCompileYaml(self): """ diff --git a/fre/make/run_fremake_script.py b/fre/make/run_fremake_script.py index d3d7c054c..df60074d7 100644 --- a/fre/make/run_fremake_script.py +++ b/fre/make/run_fremake_script.py @@ -18,11 +18,12 @@ def fremake_run(yamlfile:str, platform:str, target:str, nparallel: int = 1, njobs: int = 4, - no_parallel_checkout: Optional[bool] = None, + no_parallel_checkout: Optional[bool] = False, no_format_transfer: Optional[bool] = False, execute: Optional[bool] = False, + verbose: Optional[bool] = False, force_checkout: Optional[bool] = False, - verbose: Optional[bool] = None): + force_compile: Optional[bool] = False): """ Runs all of fre make code @@ -47,13 +48,15 @@ def fremake_run(yamlfile:str, platform:str, target:str, :type execute: bool :param force_checkout: Re-create the checkout script if changes were made to configurations :type force_checkout: bool + :param force_compile: Re-create compile script if specified + :type force_compile: bool :param verbose: Increase verbosity output :type verbose: bool """ -# if verbose: -# fre_logger.setLevel(level = logging.DEBUG) -# else: -# fre_logger.setLevel(level = logging.INFO) + if verbose: + fre_logger.setLevel(level = logging.DEBUG) + else: + fre_logger.setLevel(level = logging.INFO) # Define variables name = yamlfile.split(".")[0] @@ -100,8 +103,8 @@ def fremake_run(yamlfile:str, platform:str, target:str, #compile fre_logger.info("Running fre make: calling compile_create") compile_create(yamlfile, bm_platforms, target, njobs, nparallel, - execute, verbose) - else: + execute, verbose, force_compile) + if container_platforms: fre_logger.info("Running fre make: calling dockerfile_create") dockerfile_create(yamlfile, container_platforms, target, execute, no_format_transfer) diff --git a/fre/make/tests/compilation/test_run_fremake_builds.py b/fre/make/tests/compilation/test_run_fremake_builds.py index 1e590f6ed..fc319c182 100644 --- a/fre/make/tests/compilation/test_run_fremake_builds.py +++ b/fre/make/tests/compilation/test_run_fremake_builds.py @@ -60,7 +60,8 @@ def test_run_fremake_serial_compile(): os.environ["TEST_BUILD_DIR"] = SERIAL_TEST_PATH run_fremake_script.fremake_run(YAMLPATH, PLATFORM, TARGET, nparallel=1, njobs=1, no_parallel_checkout=False, - no_format_transfer=True, execute=True, verbose=VERBOSE) + no_format_transfer=True, execute=True, verbose=VERBOSE, + force_checkout=False, force_compile=False) assert Path( f"{SERIAL_TEST_PATH}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{TARGET[0]}/exec/{EXPERIMENT}.x").exists() @@ -71,7 +72,8 @@ def test_run_fremake_multijob_compile(): os.environ["TEST_BUILD_DIR"] = MULTIJOB_TEST_PATH run_fremake_script.fremake_run(YAMLPATH, PLATFORM, TARGET, nparallel=1, njobs=4, no_parallel_checkout=False, - no_format_transfer=False, execute=True, verbose=VERBOSE) + no_format_transfer=False, execute=True, verbose=VERBOSE, + force_checkout=False, force_compile=False) assert Path( f"{MULTIJOB_TEST_PATH}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{TARGET[0]}/exec/{EXPERIMENT}.x").exists() @@ -81,7 +83,8 @@ def test_run_fremake_container_build(): ''' checks image creation for the container build''' run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLATFORM, TARGET, nparallel=1, njobs=1, no_parallel_checkout=True, - no_format_transfer=False, execute=True, verbose=VERBOSE) + no_format_transfer=False, execute=True, verbose=VERBOSE, + force_checkout=False, force_compile=False) assert Path("null_model_full-debug.sif").exists() @pytest.mark.skipif(not can_container, reason="missing podman/apptainer") @@ -90,7 +93,8 @@ def test_run_fremake_container_build_specified_out(): os.environ["TEST_BUILD_DIR"] = CONTAINER_BUILD_TEST_PATH run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLATFORM_2, TARGET, nparallel=1, njobs=1, no_parallel_checkout=True, - no_format_transfer=False, execute=True, verbose=VERBOSE) + no_format_transfer=False, execute=True, verbose=VERBOSE, + force_checkout=False, force_compile=False) assert Path( f"{CONTAINER_BUILD_TEST_PATH}/fremake_canopy/test/null_model_full-debug.sif").exists() @@ -101,7 +105,8 @@ def test_run_fremake_container_build_notransfer(): os.remove("createContainer.sh") run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLATFORM, TARGET, nparallel=1, njobs=1, no_parallel_checkout=True, - no_format_transfer=True, execute=True, verbose=VERBOSE) + no_format_transfer=True, execute=True, verbose=VERBOSE, + force_checkout=False, force_compile=False) def test_run_fremake_cleanup(): ''' removes directories created by the test and checks to make sure they're gone ''' @@ -124,7 +129,8 @@ def test_run_fremake_container_build_fail(): # Create the createContainer.sh script but do not run run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLATFORM, TARGET, nparallel=1, njobs=1, no_parallel_checkout=True, - no_format_transfer=False, execute=False, verbose=VERBOSE) + no_format_transfer=False, execute=False, verbose=VERBOSE, + force_checkout=False, force_compile=False) assert Path(f"{currPath}/createContainer.sh").exists() # Alter script to fail diff --git a/fre/make/tests/test_create_checkout.py b/fre/make/tests/test_create_checkout.py index f1a0c3110..3c1fe63a6 100644 --- a/fre/make/tests/test_create_checkout.py +++ b/fre/make/tests/test_create_checkout.py @@ -140,11 +140,12 @@ def test_bm_checkout_force_checkout(caplog, monkeypatch): execute = False, force_checkout = True) + checkout_script = f"{OUT}/fremake_canopy/test/null_model_full/src/checkout.sh" # Check it exists, check output, check content - assert all([Path(f"{OUT}/fremake_canopy/test/null_model_full/src/checkout.sh").exists(), + assert all([Path(checkout_script).exists(), "Checkout script PREVIOUSLY created" in caplog.text, "*** REMOVING CHECKOUT SCRIPT ***" in caplog.text, - "Checkout script created" in caplog.text]) + f"Checkout script created: {checkout_script}" in caplog.text]) # Check one expected line is now populating the re-created checkout script expected_line = f"({EXPECTED_LINE}) &" @@ -194,7 +195,7 @@ def test_container_checkout_force_checkout(caplog): assert all([Path(f"{mock_checkout}/checkout.sh").exists(), "Checkout script PREVIOUSLY created" in caplog.text, "*** REMOVING CHECKOUT SCRIPT ***" in caplog.text, - "Checkout script created in ./tmp" in caplog.text]) + f"Checkout script created: ./tmp/{CONTAINER_PLATFORM[0]}/checkout.sh" in caplog.text]) # Check for an expected line that should be populating the re-created checkout script # Check no parenthesis (no parallel checkouts) diff --git a/fre/make/tests/test_create_compile.py b/fre/make/tests/test_create_compile.py index 292196231..7cf520e19 100644 --- a/fre/make/tests/test_create_compile.py +++ b/fre/make/tests/test_create_compile.py @@ -1,7 +1,6 @@ """ Test fre make compile-script """ -import os import shutil import pytest from pathlib import Path @@ -32,6 +31,13 @@ else: Path(OUT).mkdir(parents=True,exist_ok=True) +@pytest.fixture(name="stdout_fixture", autouse=True) +def test_out_fixture(monkeypatch): + """ + """ + # Set environment variable for use in ci.gnu platform + monkeypatch.setenv("TEST_BUILD_DIR", OUT) + def test_modelyaml_exists(): """ Check the model yaml exists @@ -50,13 +56,10 @@ def test_platformyaml_exists(): """ assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/platforms.yaml").exists() -def test_compile_creation(): +def test_compile_creation(stdout_fixture): """ Check for the creation of the compile script """ - # Set environment variable for use in ci.gnu platform - os.environ["TEST_BUILD_DIR"] = OUT - plat = PLATFORM[0] targ = TARGET[0] yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" @@ -68,19 +71,47 @@ def test_compile_creation(): njobs = 4, nparallel = 1, execute = False, - verbose = False) + verbose = False, + force_compile = False) # Check for creation of compile script assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/compile.sh").exists() -def test_compile_executable_failure(): +def test_force_compile_creation(caplog, stdout_fixture): + """ + """ + plat = PLATFORM[0] + targ = TARGET[0] + yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" + + # Check compile script still exists from last test + assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/compile.sh").exists() + + # 2nd compile script creation + create_compile_script.compile_create(yamlfile = yamlfile_path, + platform = PLATFORM, + target = TARGET, + njobs = 4, + nparallel = 1, + execute = False, + verbose = False, + force_compile = True) + + + # Check for re-creation of compile script + print(caplog.text) + compile_script = f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/compile.sh" + assert Path(compile_script).exists() + assert f"Compile script PREVIOUSLY created: {compile_script}" in caplog.text + assert "*** REMOVING COMPILE SCRIPT ***" in caplog.text + assert "Compile scripts available/generated with specified platform-target combination" in caplog.text + assert f" - {compile_script}" in caplog.text + +def test_compile_executable_failure(stdout_fixture): """ Check for the failure in execution of the compile script. Fails because it would need the makefile and checked out source code. """ - # Set environment variable for use in ci.gnu platform - os.environ["TEST_BUILD_DIR"] = OUT - plat = PLATFORM[0] targ = TARGET[0] yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" @@ -92,7 +123,8 @@ def test_compile_executable_failure(): njobs = 4, nparallel = 1, execute = True, - verbose = False) + verbose = False, + force_compile = False) # Check for creation of compile script, FMS directory, # log.compile file, the executable @@ -102,14 +134,11 @@ def test_compile_executable_failure(): assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{plat}-{targ}/exec/null_model_full.x").exists() == False @pytest.mark.xfail(raises=ValueError) -def test_bad_platform(): +def test_bad_platform(stdout_fixture): """ Check for the failure of compile script creation due to a bad platform passed. """ - # Set environment variable for use in ci.gnu platform - os.environ["TEST_BUILD_DIR"] = OUT - yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" # Create the compile script @@ -119,16 +148,14 @@ def test_bad_platform(): njobs = 4, nparallel = 1, execute = False, - verbose = False) + verbose = False, + force_compile = False) -def test_bad_platform_compilelog(): +def test_bad_platform_compilelog(stdout_fixture): """ Check that compile log still created from the failure of compile script creation due to a bad platform passed. """ - # Set environment variable for use in ci.gnu platform - os.environ["TEST_BUILD_DIR"] = OUT - yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" try: @@ -139,19 +166,17 @@ def test_bad_platform_compilelog(): njobs = 4, nparallel = 1, execute = False, - verbose = False) + verbose = False, + force_compile = False) except: assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{BAD_PLATFORM}-{TARGET}/exec/log.compile") @pytest.mark.xfail(raises=ValueError) -def test_bad_target(): +def test_bad_target(stdout_fixture): """ Check for the failure of compile script creation due to a bad target passed. """ - # Set environment variable for use in ci.gnu platform - os.environ["TEST_BUILD_DIR"] = OUT - yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" # Create the compile script @@ -161,16 +186,14 @@ def test_bad_target(): njobs = 4, nparallel = 1, execute = False, - verbose = False) + verbose = False, + force_compile = False) -def test_bad_target_compilelog(): +def test_bad_target_compilelog(stdout_fixture): """ Check that compile log still created from the failure of compile script creation due to a bad target passed. """ - # Set environment variable for use in ci.gnu platform - os.environ["TEST_BUILD_DIR"] = OUT - yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" try: @@ -181,17 +204,15 @@ def test_bad_target_compilelog(): njobs = 4, nparallel = 1, execute = False, - verbose = False) + verbose = False, + force_compile = False) except: assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{BAD_PLATFORM}-{TARGET}/exec/log.compile") -def test_multi_target(): +def test_multi_target(stdout_fixture): """ Check for the creation of the compile script for each target passed """ - # Set environment variable for use in ci.gnu platform - os.environ["TEST_BUILD_DIR"] = OUT - yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" # Create the compile script @@ -201,7 +222,8 @@ def test_multi_target(): njobs = 4, nparallel = 1, execute = False, - verbose = False) + verbose = False, + force_compile = False) assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{PLATFORM[0]}-{MULTI_TARGET[0]}/exec/compile.sh").exists() assert Path(f"{OUT}/fremake_canopy/test/null_model_full/{PLATFORM[0]}-{MULTI_TARGET[1]}/exec/compile.sh").exists() diff --git a/fre/make/tests/test_run_fremake.py b/fre/make/tests/test_run_fremake.py index 63a58ad98..649037fe5 100644 --- a/fre/make/tests/test_run_fremake.py +++ b/fre/make/tests/test_run_fremake.py @@ -56,21 +56,24 @@ def test_bad_platform_option(): ''' test run-fremake with a invalid platform option''' run_fremake_script.fremake_run(YAMLPATH, BADOPT, TARGET, nparallel=False, njobs=1, no_parallel_checkout=False, - no_format_transfer=False, execute=False, verbose=VERBOSE) + no_format_transfer=False, execute=False, verbose=VERBOSE, + force_checkout=False, force_compile=False) @pytest.mark.xfail() def test_bad_target_option(): ''' test run-fremake with a invalid target option''' run_fremake_script.fremake_run(YAMLPATH, PLATFORM, BADOPT, nparallel=False, njobs=1, no_parallel_checkout=False, - no_format_transfer=False, execute=False, verbose=VERBOSE) + no_format_transfer=False, execute=False, verbose=VERBOSE, + force_checkout=False, force_compile=False) @pytest.mark.xfail() def test_bad_yamlpath_option(): ''' test run-fremake with a invalid target option''' run_fremake_script.fremake_run(BADOPT[0], PLATFORM, TARGET, nparallel=False, njobs=1, no_parallel_checkout=False, - no_format_transfer=False, execute=False, verbose=VERBOSE) + no_format_transfer=False, execute=False, verbose=VERBOSE, + force_checkout=False, force_compile=False) # tests script/makefile creation without executing (serial compile) # first test runs the run-fremake command, subsequent tests check for creation of scripts @@ -79,7 +82,8 @@ def test_run_fremake_serial(): os.environ["TEST_BUILD_DIR"] = SERIAL_TEST_PATH run_fremake_script.fremake_run(YAMLPATH, PLATFORM, TARGET, nparallel=False, njobs=1, no_parallel_checkout=False, - no_format_transfer=False, execute=False, verbose=VERBOSE) + no_format_transfer=False, execute=False, verbose=VERBOSE, + force_checkout=False, force_compile=False) def test_run_fremake_compile_script_creation_serial(): ''' check for compile script creation from previous test ''' @@ -102,7 +106,8 @@ def test_run_fremake_multijob(): os.environ["TEST_BUILD_DIR"] = MULTIJOB_TEST_PATH run_fremake_script.fremake_run(YAMLPATH, PLATFORM, TARGET, nparallel=True, njobs=4, no_parallel_checkout=True, - no_format_transfer=False, execute=False, verbose=VERBOSE) + no_format_transfer=False, execute=False, verbose=VERBOSE, + force_checkout=False, force_compile=False) def test_run_fremake_compile_script_creation_multijob(): ''' check for compile script creation from previous test ''' @@ -124,7 +129,8 @@ def test_run_fremake_container(): '''run run-fremake with options for containerized build''' run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLATFORM, TARGET, nparallel=False, njobs=1, no_parallel_checkout=True, - no_format_transfer=False, execute=False, verbose=VERBOSE) + no_format_transfer=False, execute=False, verbose=VERBOSE, + force_checkout=False, force_compile=False) def test_run_fremake_build_script_creation_container(): ''' checks container build script creation from previous test ''' @@ -151,7 +157,8 @@ def test_run_fremake_container_2stage(): '''run run-fremake with options for containerized build''' run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLAT2, TARGET, nparallel=False, njobs=1, no_parallel_checkout=True, - no_format_transfer=False, execute=False, verbose=VERBOSE) + no_format_transfer=False, execute=False, verbose=VERBOSE, + force_checkout=False, force_compile=False) def test_run_fremake_build_script_creation_container_2stage(): ''' checks container build script creation from previous test ''' diff --git a/fre/tests/test_fre_cmor_cli.py b/fre/tests/test_fre_cmor_cli.py index f273a4e48..2893a2270 100644 --- a/fre/tests/test_fre_cmor_cli.py +++ b/fre/tests/test_fre_cmor_cli.py @@ -70,8 +70,8 @@ def test_cli_fre_cmor_help_and_debuglog(): assert result.exit_code == 0 assert Path("TEST_FOO_LOG.log").exists() - log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long - log_text_line_2='[DEBUG: fre.py: fre] click entry-point function call done.\n' # pylint: disable=line-too-long + log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long + log_text_line_2='[ DEBUG: fre.py: fre] click entry-point function call done.\n' # pylint: disable=line-too-long with open( "TEST_FOO_LOG.log", 'r', encoding='utf-8') as log_text: line_list=log_text.readlines() assert log_text_line_1 in line_list[0] @@ -89,7 +89,7 @@ def test_cli_fre_cmor_help_and_infolog(): assert result.exit_code == 0 assert Path("TEST_FOO_LOG.log").exists() - log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long + log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long with open( "TEST_FOO_LOG.log", 'r', encoding='utf-8') as log_text: line_list=log_text.readlines() assert log_text_line_1 in line_list[0]