diff --git a/scripts/ccpp_capgen.py b/scripts/ccpp_capgen.py index afd6a332..fae26d01 100755 --- a/scripts/ccpp_capgen.py +++ b/scripts/ccpp_capgen.py @@ -14,6 +14,7 @@ import os import logging import re +import time # CCPP framework imports from ccpp_database_obj import CCPPDatabaseObj from ccpp_datafile import generate_ccpp_datatable @@ -28,7 +29,9 @@ from metadata_table import parse_metadata_file, register_ddts, SCHEME_HEADER_TYPE from parse_tools import init_log, set_log_level, context_string from parse_tools import register_fortran_ddt_name +from parse_tools import registered_fortran_ddt_name from parse_tools import CCPPError, ParseInternalError +from ufs_depends import create_scm_build ## Capture the Framework root _SCRIPT_PATH = os.path.dirname(__file__) @@ -93,20 +96,30 @@ def delete_pathnames_from_file(capfile, logger): # end if ############################################################################### -def find_associated_fortran_file(filename, fortran_source_path): +def find_associated_fortran_file(filename, fortran_source_path=None): ############################################################################### """Find the Fortran file associated with metadata file, . Fortran files should be in . """ fort_filename = None lastdot = filename.rfind('.') - if lastdot < 0: - base = os.path.basename(filename + '.') + if (fortran_source_path is not None): + source_path = fortran_source_path + if lastdot < 0: + base = os.path.basename(filename + '.') + else: + base = os.path.basename(filename[0:lastdot+1]) + # end if else: - base = os.path.basename(filename[0:lastdot+1]) + source_path = '' + if lastdot < 0: + base = filename + '.' + else: + base = filename[0:lastdot+1] + # end if # end if for extension in _FORTRAN_FILENAME_EXTENSIONS: - test_name = os.path.join(fortran_source_path, base + extension) + test_name = os.path.join(source_path, base + extension) if os.path.exists(test_name): fort_filename = test_name break @@ -114,11 +127,43 @@ def find_associated_fortran_file(filename, fortran_source_path): # end for if fort_filename is None: emsg = f"Cannot find Fortran file associated with '{filename}'." - emsg += f"\nfortran_src_path = '{fortran_source_path}'" + emsg += f"\nfortran_src_path = '{source_path}'" raise CCPPError(emsg) # end if return fort_filename +############################################################################### +def find_dependency_files(filename, mtables, fortran_source_path): +############################################################################### + "Find the Fortran dependency files required by " + depends = list() + for mtable in mtables: + for dependency in mtable.dependencies: + file_root = find_file_root(filename) + if mtable.dependencies_path: + file_root = os.path.join(file_root, mtable.dependencies_path) + # end if + depend = find_associated_fortran_file(os.path.join(file_root, dependency)) + if (depend not in depends): + depends.append(depend) + # end if + # end for + # end for + return depends + +############################################################################### +def find_file_root(filename): +############################################################################### + "Return root directory of filename, if any" + file_root = None + last_slash = filename.rfind('/') + if last_slash < 0: + file_root = '' + else: + file_root = filename[0:last_slash] + # end if + return file_root + ############################################################################### def create_kinds_file(run_env, output_dir): ############################################################################### @@ -258,7 +303,20 @@ def compare_fheader_to_mheader(meta_header, fort_header, logger): errors_found = '' title = meta_header.title mht = meta_header.header_type - fht = fort_header.header_type + # Skip comparison for registered fortran DDTs. + if fort_header is None: + if registered_fortran_ddt_name(title): + logger.info('Skipping fortran/metadata comparison for {}'.format(title)) + return + else: + # We should never get here. + errmsg = 'No fortran header for {} of type {}' + ctx = meta_header.start_context() + raise CCPPError(errmsg.format(title, meta_header.header_type)) + # end if + else: + fht = fort_header.header_type + # end if if mht != fht: # Special case, host metadata can be in a Fortran module or scheme if (mht != 'host') or (fht not in ('module', SCHEME_HEADER_TYPE)): @@ -431,10 +489,13 @@ def check_fortran_against_metadata(meta_headers, fort_headers, # end if # end for if fheader is None: - tlist = '\n '.join([x.title for x in fort_headers]) - logger.debug("CCPP routines in {}:{}".format(ffilename, tlist)) - errmsg = "No matching Fortran routine found for {} in {}" - raise CCPPError(errmsg.format(mtitle, ffilename)) + # Exception for registerd fortran DDT. + if not registered_fortran_ddt_name(mtitle): + tlist = '\n '.join([x.title for x in fort_headers]) + logger.debug("CCPP routines in {}:{}".format(ffilename, tlist)) + errmsg = "No matching Fortran routine found for {} in {}" + raise CCPPError(errmsg.format(mtitle, ffilename)) + # end if # end if header_dict[mheader] = fheader # end if @@ -457,7 +518,15 @@ def check_fortran_against_metadata(meta_headers, fort_headers, errors_found = '' for mheader in header_dict: fheader = header_dict[mheader] - errors_found += compare_fheader_to_mheader(mheader, fheader, logger) + if fheader is None: + # Exception for registerd fortran DDT. + if registered_fortran_ddt_name(mheader.title): + logger.info('Skipping fortran/metadata comparison for {}'.format(mheader.title)) + return + # end if + else: + errors_found += compare_fheader_to_mheader(mheader, fheader, logger) + # end if # end for if errors_found: num_errors = len(re.findall(r'\n', errors_found)) + 1 @@ -483,7 +552,7 @@ def duplicate_item_error(title, filename, itype, orig_item): ############################################################################### def parse_host_model_files(host_filenames, host_name, run_env, - known_ddts=list()): + known_ddts=list(), debug=None): ############################################################################### """ Gather information from host files (e.g., DDTs, registry) and @@ -491,14 +560,17 @@ def parse_host_model_files(host_filenames, host_name, run_env, """ header_dict = {} table_dict = {} + fort_files = list() + mod_files = list() + depend_files = list() logger = run_env.logger for filename in host_filenames: logger.info('Reading host model data from {}'.format(filename)) # parse metadata file - mtables = parse_metadata_file(filename, known_ddts, run_env) + mtables,mtitles = parse_metadata_file(filename, known_ddts, run_env) fortran_source_path = mtables[0].fortran_source_path - fort_file = find_associated_fortran_file(filename, fortran_source_path) - ftables, _ = parse_fortran_file(fort_file, run_env) + fort_file = find_associated_fortran_file(filename, fortran_source_path = fortran_source_path) + ftables, mod_file, additional_routines = parse_fortran_file(fort_file, run_env) # Check Fortran against metadata (will raise an exception on error) mheaders = list() for sect in [x.sections() for x in mtables]: @@ -508,8 +580,22 @@ def parse_host_model_files(host_filenames, host_name, run_env, for sect in [x.sections() for x in ftables]: fheaders.extend(sect) # end for - check_fortran_against_metadata(mheaders, fheaders, - filename, fort_file, logger) + # Compare Host metadata tables (DEBUG mode only). + if (debug is not None): + logger.info('Comparing {}, to {}.'.format(fort_file,filename)) + check_fortran_against_metadata(mheaders, fheaders, + filename, fort_file, logger) + # end if + # Check for host dependencies (will raise error if reqired + # dependency file not found) + if run_env.ccpp_cfgfile: + depends = find_dependency_files(filename, mtables, fortran_source_path) + for depend in depends: + if (depend not in depend_files): + depend_files.append(depend) + # end if + # end for + # end if # Check for duplicate tables, then add to dict for table in mtables: if table.table_name in table_dict: @@ -529,6 +615,13 @@ def parse_host_model_files(host_filenames, host_name, run_env, header_dict[header.title] = header if header.header_type == 'ddt': known_ddts.append(header.title) + # Store fortran file + if fort_file: + if not (fort_file in fort_files): + fort_files.append(fort_file) + mod_files.append(mod_file+'.mod') + # end if + # end if # end if # end for # end for @@ -536,11 +629,11 @@ def parse_host_model_files(host_filenames, host_name, run_env, host_name = None # end if host_model = HostModel(table_dict, host_name, run_env) - return host_model + return host_model, fort_files, mod_files, depend_files ############################################################################### def parse_scheme_files(scheme_filenames, run_env, skip_ddt_check=False, - known_ddts=list(), relative_source_path=False): + known_ddts=list(), relative_source_path=False, debug=None): ############################################################################### """ Gather information from scheme files (e.g., init, run, and finalize @@ -548,16 +641,19 @@ def parse_scheme_files(scheme_filenames, run_env, skip_ddt_check=False, """ table_dict = {} # Duplicate check and for dependencies processing header_dict = {} # To check for duplicates + fort_files = list() + depend_files = list() + table_names = list() logger = run_env.logger for filename in scheme_filenames: logger.info('Reading CCPP schemes from {}'.format(filename)) # parse metadata file - mtables = parse_metadata_file(filename, known_ddts, run_env, - skip_ddt_check=skip_ddt_check, - relative_source_path=relative_source_path) + mtables, mtitles = parse_metadata_file(filename, known_ddts, run_env, + skip_ddt_check=skip_ddt_check, + relative_source_path=relative_source_path) fortran_source_path = mtables[0].fortran_source_path - fort_file = find_associated_fortran_file(filename, fortran_source_path) - ftables, additional_routines = parse_fortran_file(fort_file, run_env) + fort_file = find_associated_fortran_file(filename, fortran_source_path = fortran_source_path) + ftables, mod_file, additional_routines = parse_fortran_file(fort_file, run_env) # Check Fortran against metadata (will raise an exception on error) mheaders = list() for sect in [x.sections() for x in mtables]: @@ -567,9 +663,23 @@ def parse_scheme_files(scheme_filenames, run_env, skip_ddt_check=False, for sect in [x.sections() for x in ftables]: fheaders.extend(sect) # end for - check_fortran_against_metadata(mheaders, fheaders, - filename, fort_file, logger, - fortran_routines=additional_routines) + # Compare Scheme metadata tables (DEBUG mode only). + if (debug is not None): + logger.info('Comparing {}, to {}.'.format(fort_file,filename)) + check_fortran_against_metadata(mheaders, fheaders, + filename, fort_file, logger, + fortran_routines=additional_routines) + # end if + # Check for scheme dependencies (will raise error if reqired + # dependency file not found) + if run_env.ccpp_cfgfile: + depends = find_dependency_files(filename, mtables, fortran_source_path) + for depend in depends: + if not (depend in depend_files): + depend_files.append(depend) + # end if + # end for + # end if # Check for duplicate tables, then add to dict for table in mtables: if table.table_name in table_dict: @@ -577,6 +687,7 @@ def parse_scheme_files(scheme_filenames, run_env, skip_ddt_check=False, table.table_type, table_dict[header.title]) else: table_dict[table.table_name] = table + table_names.append(table.table_name) # end if # end for # Check for duplicate headers, then add to dict @@ -589,11 +700,17 @@ def parse_scheme_files(scheme_filenames, run_env, skip_ddt_check=False, if header.header_type == 'ddt': known_ddts.append(header.title) # end if + # Store fortran file + if fort_file: + if not (fort_file in fort_files): + fort_files.append(fort_file) + # end if + # end if # end if # end for # end for - return header_dict.values(), table_dict + return header_dict.values(), table_dict, fort_files, depend_files ############################################################################### def clean_capgen(cap_output_file, logger): @@ -615,6 +732,10 @@ def capgen(run_env, return_db=False): ############################################################################### """Parse indicated host, scheme, and suite files. Generate code to allow host model to run indicated CCPP suites.""" + timing_info = [] + timing_label = [] + timing_info.append(time.time()) + timing_label.append('Start') ## A few sanity checks ## Make sure output directory is legit if os.path.exists(run_env.output_dir): @@ -630,10 +751,14 @@ def capgen(run_env, return_db=False): # Try to create output_dir (let it crash if it fails) os.makedirs(run_env.output_dir) # end if + timing_info.append(time.time()) + timing_label.append('Sanity checks') # Pre-register base CCPP DDT types: for ddt_name in _CCPP_FRAMEWORK_DDT_TYPES: register_fortran_ddt_name(ddt_name) # end for + timing_info.append(time.time()) + timing_label.append('Register DDT names') src_dir = os.path.join(_FRAMEWORK_ROOT, "src") host_files = run_env.host_files host_name = run_env.host_name @@ -641,33 +766,49 @@ def capgen(run_env, return_db=False): # We need to create three lists of files, hosts, schemes, and SDFs host_files = create_file_list(run_env.host_files, ['meta'], 'Host', run_env.logger) + timing_info.append(time.time()) + timing_label.append('Create Host file lists') # The host model needs to know about the constituents module const_mod = os.path.join(_SRC_ROOT, "ccpp_constituent_prop_mod.meta") if const_mod not in host_files: host_files.append(const_mod) # end if + timing_info.append(time.time()) + timing_label.append('Add constituent file to Host file lists') scheme_files = create_file_list(run_env.scheme_files, ['meta'], 'Scheme', run_env.logger) + timing_info.append(time.time()) + timing_label.append('Create Scheme file lists') sdfs = create_file_list(run_env.suites, ['xml'], 'Suite', run_env.logger) check_for_writeable_file(run_env.datatable_file, "Cap output datatable") + timing_info.append(time.time()) + timing_label.append('Create SDF file lists') ##XXgoldyXX: Temporary warning if run_env.generate_docfiles: raise CCPPError("--generate-docfiles not yet supported") # end if # The host model may depend on suite DDTs scheme_ddts = register_ddts(scheme_files) + timing_info.append(time.time()) + timing_label.append('Register DDTs') # Handle the host files - host_model = parse_host_model_files(host_files, host_name, run_env, - known_ddts=scheme_ddts) + host_model, host_ffiles, host_mods, host_depends = \ + parse_host_model_files(host_files, host_name, run_env, known_ddts=scheme_ddts, debug=run_env.debug) + timing_info.append(time.time()) + timing_label.append('Parse Host files') # Next, parse the scheme files # We always need to parse the constituent DDTs const_prop_mod = os.path.join(src_dir, "ccpp_constituent_prop_mod.meta") if const_prop_mod not in scheme_files: scheme_files = [const_prop_mod] + scheme_files # end if + timing_info.append(time.time()) + timing_label.append('Parse constituent file') host_ddts = register_ddts(host_files) - scheme_headers, scheme_tdict = parse_scheme_files(scheme_files, run_env, - known_ddts=host_ddts) + scheme_headers, scheme_tdict, scheme_ffiles, scheme_depends = \ + parse_scheme_files(scheme_files, run_env, known_ddts=host_ddts, debug=run_env.debug) + timing_info.append(time.time()) + timing_label.append('Parse Scheme files') if run_env.verbose: ddts = host_model.ddt_lib.keys() if ddts: @@ -708,6 +849,8 @@ def capgen(run_env, return_db=False): else: host_files = list() # end if + timing_info.append(time.time()) + timing_label.append('Write suite caps') # Create the kinds file kinds_file = create_kinds_file(run_env, outtemp_dir) # Move any changed files to output_dir and remove outtemp_dir @@ -719,15 +862,35 @@ def capgen(run_env, return_db=False): replace_paths(host_files, outtemp_dir, run_env.output_dir) kinds_file = kinds_file.replace(outtemp_dir, run_env.output_dir) # end if + timing_info.append(time.time()) + timing_label.append('Create kinds file') + + # DJS2024: Call UFS/SCM build configurations (Ultimately SCM/UFS Cmake step could be + # updated to use datatable.xml generated by Capgen.) + if run_env.ccpp_cfgfile: + static_api = run_env.output_dir + '/'+ host_model.ccpp_cap_name() + '.F90' + kinds_file = list(kinds_file.split()) + create_scm_build(run_env, scheme_ffiles, host_ffiles, scheme_depends, + host_depends, cap_filenames, host_mods, static_api, + kinds_file) # Finally, create the database of generated files and caps - # This can be directly in output_dir because it will not affect dependencies - generate_ccpp_datatable(run_env, host_model, ccpp_api, - scheme_headers, scheme_tdict, host_files, - cap_filenames, kinds_file, src_dir) - if return_db: - return CCPPDatabaseObj(run_env, host_model=host_model, api=ccpp_api) - # end if - return None + else: + generate_ccpp_datatable(run_env, host_model, ccpp_api, + scheme_headers, scheme_tdict, host_files, + cap_filenames, kinds_file, src_dir) + if return_db: + return CCPPDatabaseObj(run_env, host_model=host_model, api=ccpp_api) + # end if + # end if (DO UFS/SCM build configuration) + timing_info.append(time.time()) + timing_label.append('Build system configuration') + time_info = [0] + for index, x in enumerate(timing_info): + if index > 0: + time_info.append(timing_info[index] - timing_info[index-1]) + # end if + # end for + return time_info, timing_label ############################################################################### def _main_func(): @@ -743,8 +906,15 @@ def _main_func(): if framework_env.clean: clean_capgen(framework_env.datatable_file, framework_env.logger) else: - _ = capgen(framework_env) + timing_info, timing_label = capgen(framework_env) # end if (clean) + + print('-'*50) + for index, timing in enumerate(timing_info): + if (index > 0): + print("{} : {:.2f} seconds".format(timing_label[index].ljust(40), timing_info[index])) + # end if + # end for ############################################################################### diff --git a/scripts/ccpp_fortran_to_metadata.py b/scripts/ccpp_fortran_to_metadata.py index afb3597d..74e553ce 100755 --- a/scripts/ccpp_fortran_to_metadata.py +++ b/scripts/ccpp_fortran_to_metadata.py @@ -182,7 +182,7 @@ def parse_fortran_files(filenames, run_env, output_dir, sep, logger): for filename in filenames: logger.info('Looking for arg_tables from {}'.format(filename)) reset_standard_name_counter() - ftables, _ = parse_fortran_file(filename, run_env) + ftables, _, __ = parse_fortran_file(filename, run_env) # Create metadata filename filepath = '.'.join(os.path.basename(filename).split('.')[0:-1]) fname = filepath + '.meta' diff --git a/scripts/ddt_library.py b/scripts/ddt_library.py index 1c362108..50f4df2d 100644 --- a/scripts/ddt_library.py +++ b/scripts/ddt_library.py @@ -280,9 +280,10 @@ def collect_ddt_fields(self, var_dict, var, run_env, # end if dvtype = dvar.get_prop_value('type') if (dvar.is_ddt()) and (dvtype in self): - # If DDT in our library, we need to add sub-fields recursively. + # If DDT in our library, we need to add sub-fields recursively, + # unless sub-fields already added to library previously. subddt = self[dvtype] - self.collect_ddt_fields(var_dict, dvar, run_env, parent=var, ddt=subddt) + self.collect_ddt_fields(var_dict, dvar, run_env, parent=var, ddt=subddt, skip_duplicates=skip_duplicates) # end if # add_variable only checks the current dictionary. By default, # for a DDT, the variable also cannot be in our parent diff --git a/scripts/fortran_tools/fortran_write.py b/scripts/fortran_tools/fortran_write.py index e147b48c..2d53b110 100644 --- a/scripts/fortran_tools/fortran_write.py +++ b/scripts/fortran_tools/fortran_write.py @@ -5,6 +5,8 @@ """ import math +import sys +sys.setrecursionlimit(1500) class FortranWriter: """Class to turn output into properly continued and indented Fortran code @@ -25,7 +27,7 @@ class FortranWriter: __LINE_FILL = 97 # Target line length - __LINE_MAX = 130 # Max line length + __LINE_MAX = 195 # Max line length # CCPP copyright statement to be included in all generated Fortran files __COPYRIGHT = '''! diff --git a/scripts/fortran_tools/parse_fortran_file.py b/scripts/fortran_tools/parse_fortran_file.py index 9808f6c8..2c621b9b 100644 --- a/scripts/fortran_tools/parse_fortran_file.py +++ b/scripts/fortran_tools/parse_fortran_file.py @@ -1035,7 +1035,7 @@ def parse_module(pobj, statements, run_env): if errors: raise CCPPError('\n'.join(errors)) # end if - return statements, mtables, additional_subroutines + return statements, mtables, mod_name,additional_subroutines ######################################################################## @@ -1060,14 +1060,14 @@ def parse_fortran_file(filename, run_env): elif _MODULE_RE.match(statement) is not None: # push statement back so parse_module can use it statements.insert(0, statement) - statements, ptables, additional_routines = parse_module(pobj, statements, run_env) + statements, ptables, mod_name, additional_routines = parse_module(pobj, statements, run_env) mtables.extend(ptables) # end if if (statements is not None) and (len(statements) == 0): statements = read_statements(pobj) # end if # end while - return mtables, additional_routines + return mtables, mod_name, additional_routines ######################################################################## diff --git a/scripts/framework_env.py b/scripts/framework_env.py index 88c9c204..a12eb000 100644 --- a/scripts/framework_env.py +++ b/scripts/framework_env.py @@ -26,7 +26,7 @@ def __init__(self, logger, ndict=None, verbose=0, clean=False, preproc_directives=[], generate_docfiles=False, host_name='', kind_types=[], use_error_obj=False, force_overwrite=False, output_root=os.getcwd(), ccpp_datafile="datatable.xml", - debug=False): + debug=False, ccpp_cfgfile=None): """Initialize a new CCPPFrameworkEnv object from the input arguments. is a dict with the parsed command-line arguments (or a dictionary created with the necessary arguments). @@ -199,6 +199,13 @@ def __init__(self, logger, ndict=None, verbose=0, clean=False, self.__datatable_file = os.path.join(self.output_dir, self.datatable_file) # end if + # Are we provided a CCPP configuration file? + if ndict and ('ccpp_cfgfile' in ndict): + self.__ccpp_cfgfile = ndict['ccpp_cfgfile'] + del ndict['ccpp_cfgfile'] + else: + self.__ccpp_cfgfile = ccpp_cfgfile + # endif # Enable or disable variable allocation checks if ndict and ('debug' in ndict): self.__debug = ndict['debug'] @@ -319,6 +326,12 @@ def datatable_file(self): CCPPFrameworkEnv object.""" return self.__datatable_file + @property + def ccpp_cfgfile(self): + """Return the property for this + CCPPFrameworkEnv object.""" + return self.__ccpp_cfgfile + @property def debug(self): """Return the property for this @@ -368,6 +381,9 @@ def parse_command_line(args, description, logger=None): default="datatable.xml", help="Filename for information on content generated by the CCPP Framework") + parser.add_argument('--ccpp-cfgfile', type=str, default=None, + help='path to CCPP prebuild configuration file') + parser.add_argument("--output-root", type=str, metavar='', default=os.getcwd(), diff --git a/scripts/host_cap.py b/scripts/host_cap.py index b06906fe..91848f43 100644 --- a/scripts/host_cap.py +++ b/scripts/host_cap.py @@ -512,8 +512,39 @@ def suite_part_call_list(host_model, const_dict, suite_part, subst_loop_vars, # End if if stdname not in CCPP_CONSTANT_VARS: lname = var_dict.var_call_string(hvar, loop_vars=loop_vars) + # DJS2025: Check to see if call_string for variable contains any + # array references with a standard_name, . If so, replace + # with full reference, )>. + str_start = 0 + while True: + dimdA = lname.find('(',str_start) + dimdB = lname.find(')',str_start) + if (dimdA > 0 and dimdB > 0): + if (dimdB > dimdA): + sname_sub = lname[dimdA+1:dimdB] + hvar_sub = vdict.find_variable(standard_name=sname_sub, any_scope=True) + if (hvar_sub): + lnameA = lname[0:dimdA+1] + lnameB = lname[dimdB::] + lname = lnameA + var_dict.var_call_string(hvar_sub, loop_vars=loop_vars) + lnameB + # end if + # end if + else: + break + # end if + str_start += dimdB+1 + # end while + # Also check if {sp_lname} has an array reference to a loop varaible, if so, + # remove array reference from LHS call_string. + if stdname in CCPP_LOOP_VAR_STDNAMES: + dimdA = sp_lname.find('(',0) + if (dimdA > 0): + sp_lname = sp_lname[0:dimdA] + # endif + # endif hmvars.append(f"{sp_lname}={lname}") - # End if + # end if + # end if # End for return ', '.join(hmvars) @@ -543,14 +574,23 @@ def write_host_cap(host_model, api, module_name, output_dir, run_env): mspc = (maxmod - len(mod[0]))*' ' cap.write("use {}, {}only: {}".format(mod[0], mspc, mod[1]), 1) # End for + # Write out any suite part use statements + max_suite_len = host_model.ddt_lib.max_mod_name_len + for suite in api.suites: + max_suite_len = max(max_suite_len, len(suite.module)) + mspc = (max_suite_len - len(suite.module))*' ' + for stage in CCPP_STATE_MACH.transitions(): + spart_list = suite_part_list(suite, stage) + for _, spart in sorted(enumerate(spart_list)): + stmt = "use {}, {}only: {}" + cap.write(stmt.format(suite.module, mspc, spart.name), 1) + # end for + # end for + # end for mspc = ' '*(maxmod - len(CONST_DDT_MOD)) cap.write(f"use {CONST_DDT_MOD}, {mspc}only: {CONST_DDT_NAME}", 1) cap.write(f"use {CONST_DDT_MOD}, {mspc}only: {CONST_PROP_TYPE}", 1) cap.write_preamble() - max_suite_len = host_model.ddt_lib.max_mod_name_len - for suite in api.suites: - max_suite_len = max(max_suite_len, len(suite.module)) - # End for cap.comment("Public Interfaces", 1) # CCPP_STATE_MACH.transitions represents the host CCPP interface for stage in CCPP_STATE_MACH.transitions(): @@ -667,15 +707,7 @@ def write_host_cap(host_model, api, module_name, output_dir, run_env): cap.write(_SUBHEAD.format(api_vars=api_vlist, host_model=host_model.name, stage=stage), 1) - # Write out any suite part use statements - for suite in api.suites: - mspc = (max_suite_len - len(suite.module))*' ' - spart_list = suite_part_list(suite, stage) - for _, spart in sorted(enumerate(spart_list)): - stmt = "use {}, {}only: {}" - cap.write(stmt.format(suite.module, mspc, spart.name), 2) - # End for - # End for + # Write out any host model DDT input var use statements host_model.ddt_lib.write_ddt_use_statements(hdvars, cap, 2, pad=max_suite_len) diff --git a/scripts/host_model.py b/scripts/host_model.py index c3beb447..91cbfb80 100644 --- a/scripts/host_model.py +++ b/scripts/host_model.py @@ -59,9 +59,13 @@ def __init__(self, meta_tables, name_in, run_env): lname = var.get_prop_value('local_name') self.__var_locations[lname] = modname self.ddt_lib.check_ddt_type(var, header, lname=lname) + # DDTs may be used more than once by the host, so + # skip duplicate sub-field entries. These fields were + # encountered earlier and already added to library. if var.is_ddt(): self.ddt_lib.collect_ddt_fields(self.__ddt_dict, var, - run_env) + run_env, + skip_duplicates=True) # End if # End for elif header.header_type == 'host': @@ -73,8 +77,12 @@ def __init__(self, meta_tables, name_in, run_env): self.add_variable(var, run_env) self.ddt_lib.check_ddt_type(var, header) if var.is_ddt(): + # DDTs may be used more than once by the host, so + # skip duplicate sub-field entries. These fields were + # encountered earlier and already added to library. self.ddt_lib.collect_ddt_fields(self.__ddt_dict, var, - run_env) + run_env, + skip_duplicates=True) # End if # End for loop_vars = header.variable_list(std_vars=False, diff --git a/scripts/metadata2html.py b/scripts/metadata2html.py index 7e4a540d..40f52fbc 100755 --- a/scripts/metadata2html.py +++ b/scripts/metadata2html.py @@ -99,7 +99,7 @@ def convert_to_html(filename_in, outdir, logger, run_env): if not os.path.isfile(filename_in): raise Exception("Metadata file {} not found".format(filename_in)) logger.info("Converting file {} to HTML".format(filename_in)) - metadata_headers = parse_metadata_file(filename_in, + metadata_headers, metadata_titles = parse_metadata_file(filename_in, known_ddts=registered_fortran_ddt_names(), run_env=run_env) for metadata_header in metadata_headers: diff --git a/scripts/metadata_parser.py b/scripts/metadata_parser.py index 0b25538f..65dc0aa7 100755 --- a/scripts/metadata_parser.py +++ b/scripts/metadata_parser.py @@ -112,7 +112,7 @@ def read_new_metadata(filename, module_name, table_name, scheme_name = None, sub if filename in NEW_METADATA_SAVE.keys(): new_metadata_headers = NEW_METADATA_SAVE[filename] else: - new_metadata_headers = parse_metadata_file(filename, known_ddts=registered_fortran_ddt_names(), + new_metadata_headers, new_metadata_titles = parse_metadata_file(filename, known_ddts=registered_fortran_ddt_names(), run_env=_DUMMY_RUN_ENV) NEW_METADATA_SAVE[filename] = new_metadata_headers diff --git a/scripts/metadata_table.py b/scripts/metadata_table.py index 5e4d2d7c..550cb622 100755 --- a/scripts/metadata_table.py +++ b/scripts/metadata_table.py @@ -140,6 +140,7 @@ from parse_tools import check_fortran_ref, check_fortran_id from parse_tools import check_fortran_intrinsic from parse_tools import register_fortran_ddt_name, unique_standard_name +from parse_tools import registered_fortran_ddt_name ######################################################################## @@ -222,7 +223,7 @@ def parse_metadata_file(filename, known_ddts, run_env, skip_ddt_check=False, rel context=parse_obj) # end if # end while - return meta_tables + return meta_tables, table_titles ######################################################################## @@ -979,7 +980,7 @@ def parse_variable(self, curr_line, known_ddts, skip_ddt_check=False): if skip_ddt_check: register_fortran_ddt_name(pval_str) # end if - pval = pval_str.lower() + pval = pval_str pname = 'ddt_type' else: errmsg = "Unknown DDT type, {}".format(pval_str) diff --git a/scripts/metavar.py b/scripts/metavar.py index cafdbf9f..a3045e99 100755 --- a/scripts/metavar.py +++ b/scripts/metavar.py @@ -399,6 +399,9 @@ def compatible(self, other, run_env, is_tend=False): # end if return compat + def adjust_optional(self, src_var): + self._prop_dict['optional'] = True + def adjust_intent(self, src_var): """Add an intent to this Var or adjust its existing intent. Note: An existing intent can only be adjusted to 'inout' @@ -701,7 +704,7 @@ def call_string(self, var_dict, loop_vars=None): lname = "" for item in dim.split(':'): if item: - dvar = var_dict.find_variable(standard_name=item, + dvar = var_dict.find_variable(standard_name=item.lower(), any_scope=False) if dvar is None: try: @@ -720,6 +723,17 @@ def call_string(self, var_dict, loop_vars=None): if iname is not None: lname = lname + isep + iname isep = ':' + # DJS2025: HACK. Check if dimension provided by host? + elif item == "ccpp_chunk_number": + errmsg = "SWALES: Cannot find {} in {}. Use {} instead" + lname = "ccpp_cfg%chunk_no" + ctx = context_string(self.context) + #self.__run_env.logger.warning(errmsg.format(item,ctx,lname)) + elif item == "ccpp_chunk_count": + errmsg = "SWALES: Cannot find {} in {}. Use {} instead" + lname = "ccpp_cfg%chunk_cnt" + ctx = context_string(self.context) + #self.__run_env.logger.warning(errmsg.format(item,ctx,lname)) else: errmsg = 'No local variable {} in {}{}' ctx = context_string(self.context) @@ -1012,14 +1026,33 @@ def conditional(self, vdicts): conditional += item except ValueError: dvar = None + # DJS2024: First, check if any schemes in the group have this + # variable as an argument. If so, no need to add to list of + # varaibles needed, vars_needed. for vdict in vdicts: - dvar = vdict.find_variable(standard_name=item, any_scope=True) # or any_scope=False ? + dvar = vdict.find_variable(standard_name=item, any_scope=False) if dvar: break + # end if + # end for + # DJS2024: If varaible is not requested by a scheme in this group, but + # is needed for optional variable quiering, add to list + # of varaibles needed, vars_needed. + if not dvar: + for vdict in vdicts: + dvar = vdict.find_variable(standard_name=item, any_scope=True) + if dvar: + vars_needed.append(dvar) + break + # end if + # end for + # end if + if not dvar: raise Exception(f"Cannot find variable '{item}' for generating conditional for '{active}'") + # end if conditional += dvar.get_prop_value('local_name') - vars_needed.append(dvar) + return (conditional, vars_needed) def write_def(self, outfile, indent, wdict, allocatable=False, target=False, @@ -1134,21 +1167,6 @@ def write_def(self, outfile, indent, wdict, allocatable=False, target=False, name=name, dims=dimstr, cspace=cspace, sname=stdname), indent) - def write_ptr_def(self, outfile, indent, name, kind, dimstr, vtype, extra_space=0): - """Write the definition line for local null pointer declaration to .""" - comma = ', ' - if kind: - dstr = "{type}({kind}){cspace}pointer :: {name}{dims}{cspace2} => null()" - cspace = comma + ' '*(extra_space + 20 - len(vtype) - len(kind)) - cspace2 = ' '*(20 -len(name) - len(dimstr)) - else: - dstr = "{type}{cspace}pointer :: {name}{dims}{cspace2} => null()" - cspace = comma + ' '*(extra_space + 22 - len(vtype)) - cspace2 = ' '*(20 -len(name) - len(dimstr)) - # end if - outfile.write(dstr.format(type=vtype, kind=kind, name=name, dims=dimstr, - cspace=cspace, cspace2=cspace2), indent) - def is_ddt(self): """Return True iff is a DDT type.""" return not self.__intrinsic @@ -1653,6 +1671,15 @@ def add_variable(self, newvar, run_env, exists_ok=False, gen_unique=False, nlname, dintent, nctx)) # end if # end if + # Check for optional argument mismatch. + # If a variable is optional to ANY Scheme in the Group, adjust + # Group definition to always as optional. + vopt = cvar.get_prop_value('optional') + dopt = newvar.get_prop_value('optional') + if vopt != dopt: + cvar.adjust_optional(newvar) + # end if + else: if self.__run_env.logger is not None: emsg = "Attempt to add incompatible variable, {} from {}" @@ -1694,7 +1721,7 @@ def add_variable(self, newvar, run_env, exists_ok=False, gen_unique=False, # end if # end if # end if - if gen_unique: + if gen_unique and (newvar_callstr == lname): new_lname = self.new_internal_variable_name(prefix=lname) newvar = newvar.clone(new_lname) # Local_name needs to be the local_name for the new @@ -2128,6 +2155,21 @@ def new_internal_variable_name(self, prefix=None, max_len=63): # end while return newvar +def write_ptr_def(outfile, indent, name, kind, dimstr, vtype, extra_space=0): + """Write the definition line for local null pointer declaration to .""" + comma = ', ' + if kind: + dstr = "{type}({kind}){cspace}pointer :: {name}{dims}{cspace2} => null()" + cspace = comma + ' '*(extra_space + 20 - len(vtype) - len(kind)) + cspace2 = ' '*(20 -len(name) - len(dimstr)) + else: + dstr = "{type}{cspace}pointer :: {name}{dims}{cspace2} => null()" + cspace = comma + ' '*(extra_space + 22 - len(vtype)) + cspace2 = ' '*(20 -len(name) - len(dimstr)) + # end if + outfile.write(dstr.format(type=vtype, kind=kind, name=name, dims=dimstr, + cspace=cspace, cspace2=cspace2), indent) + ############################################################################### # List of constant variables which are universally available diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py index b1de44b8..d512a895 100755 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -14,6 +14,7 @@ from constituents import ConstituentVarDict from framework_env import CCPPFrameworkEnv from metavar import Var, VarDictionary, VarLoopSubst +from metavar import write_ptr_def from metavar import CCPP_CONSTANT_VARS, CCPP_LOOP_VAR_STDNAMES from parse_tools import ParseContext, ParseSource, context_string from parse_tools import ParseInternalError, CCPPError @@ -142,12 +143,17 @@ def call_string(self, cldicts=None, is_func_call=False, subname=None, sub_lname_ raise CCPPError(errmsg.format(stdname, clnames)) # end if lname = dvar.get_prop_value('local_name') - # Optional variables in the caps are associated with - # local pointers of _ptr - if dvar.get_prop_value('optional'): - lname = dummy+'_ptr' + # Optional arguments in the Group caps are associated with + # local pointers . uses the local_name + # from Group's call list (cldict.find_variable). + if var.get_prop_value('optional'): + sname = dvar.get_prop_value('standard_name') + svar = cldict.find_variable(standard_name=sname, any_scope=True) + lname = svar.get_prop_value('local_name')+'_ptr' # end if else: + dvar = self.find_variable(standard_name=stdname, + any_scope=False) cldict = None aref = var.array_ref(local_name=dummy) if aref is not None: @@ -172,6 +178,7 @@ def call_string(self, cldicts=None, is_func_call=False, subname=None, sub_lname_ use_dicts = cldicts else: use_dicts = [self] + dvar = self.find_variable(standard_name=stdname) # end if run_phase = self.routine.run_phase() # We only need dimensions for suite variables in run phase @@ -467,12 +474,14 @@ def add_call_list_variable(self, newvar, exists_ok=False, emsg = f"{self.name}: Cannot have unnamed/empty string dimension" raise ParseInternalError(emsg) # end if - dvar = self.find_variable(standard_name=vardim, + dvar = self.find_variable(standard_name=vardim.lower(), any_scope=True) if dvar is None: - emsg = "{}: Could not find dimension {} in {}" - raise ParseInternalError(emsg.format(self.name, - vardim, stdname)) + if self.run_env.debug: + emsg = "{}: Could not find dimension {} in {}" + raise ParseInternalError(emsg.format(self.name, + vardim, stdname)) + # end if # end if elif self.parent is None: errmsg = 'No call_list found for {}'.format(newvar) @@ -711,8 +720,8 @@ def match_dimensions(self, need_dims, have_dims): break # end if # end if - # end if - # end for + # end for + # end if if not found_ndim: match = False reason = 'Could not find dimension, ' + neddim + ', in ' @@ -908,17 +917,9 @@ def match_variable(self, var, run_env): new_dict_dims = dict_dims match = True # end if - # If variable is defined as "inactive" by the host, ensure that - # this variable is declared as "optional" by the scheme. If - # not satisfied, return error. - host_var_active = dict_var.get_prop_value('active') - scheme_var_optional = var.get_prop_value('optional') - if (not scheme_var_optional and host_var_active.lower() != '.true.'): - errmsg = "Non optional scheme arguments for conditionally allocatable variables" - sname = dict_var.get_prop_value('standard_name') - errmsg += ", {}".format(sname) - raise CCPPError(errmsg) - # end if + # Create compatability object, containing any necessary forward/reverse + # transforms from and + compat_obj = var.compatible(dict_var, run_env) # Add the variable to the parent call tree if dict_dims == new_dict_dims: sdict = {} @@ -1277,11 +1278,14 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): # end if # Is this a conditionally allocated variable? - # If so, declare localpointer variable. This is needed to + # If so, declare local pointer variable. This is needed to # pass inactive (not present) status through the caps. if var.get_prop_value('optional'): - newvar_ptr = var.clone(var.get_prop_value('local_name')+'_ptr') - self.__optional_vars.append([dict_var, var, newvar_ptr, has_transform]) + if dict_var: + self.add_optional_var(dict_var, var, has_transform) + # end if + #newvar_ptr = var.clone(var.get_prop_value('local_name')+'_ptr') + #self.__optional_vars.append([dict_var, var, has_transform]) # end if # end for @@ -1295,6 +1299,37 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): # end if return scheme_mods + def add_optional_var(self, dict_var, var, has_transform): + """Add local pointer needed for optional variable(s) in Group Cap. Also, + add any host variables from active condition that are needed to associate + the local pointer correctly.""" + # Use Group call list and local suite dictionary. + var_dicts = [ self.__group.call_list ] + self.__group.suite_dicts() + + # We need to gather all of the variables that are part of the 'active' attribute + # conditional and add them to the group's call list. + (_, vars_needed) = dict_var.conditional(var_dicts) + for var_needed in vars_needed: + self.update_group_call_list_variable(var_needed) + # end for + + # Create new internal pointer variable, if not already created + # previously. If necessary, the same local pointer is recylced + # throughout the Group cap. + found = self.__group.find_variable(source_var=var, any_scope=False) + if not found: + lname = var.get_prop_value('local_name') + sname = var.get_prop_value('standard_name') + found = self.__group.find_variable(standard_name=sname) + if not found: + lname_ptr = lname + '_ptr' + newvar_ptr = var.clone(lname_ptr) + self.add_variable(newvar_ptr, self.run_env) + # end if + # end if + + return self.__optional_vars.append([dict_var, var, has_transform]) + def add_var_debug_check(self, var): """Add a debug check for a given variable var (host model variable, suite variable or group module variable) for this scheme. @@ -1358,7 +1393,7 @@ def add_var_debug_check(self, var): if dimensions: for dim in dimensions: if not ':' in dim: - dim_var = self.find_variable(standard_name=dim) + dim_var = self.find_variable(standard_name=dim.lower()) if not dim_var: # To allow for numerical dimensions in metadata. if not dim.isnumeric(): @@ -1369,7 +1404,7 @@ def add_var_debug_check(self, var): # end if else: (ldim, udim) = dim.split(":") - ldim_var = self.find_variable(standard_name=ldim) + ldim_var = self.find_variable(standard_name=ldim.lower()) if not ldim_var: # To allow for numerical dimensions in metadata. if not ldim.isnumeric(): @@ -1425,7 +1460,7 @@ def replace_horiz_dim_debug_check(self, dim, cldicts, var_in_call_list): raise Exception(f"No variable with standard name '{udim}' in cldicts") udim_lname = dvar.get_prop_value('local_name') # Assemble dimensions and bounds for size checking - dim_length = f'{udim_lname}-{ldim_lname}+1' + dim_length = f'abs({udim_lname}-{ldim_lname})+1' # If the variable that uses these dimensions is not in the group's call # list, then it is defined as a module variable for this group and the # dimensions run from ldim to udim, otherwise from 1:dim_length. @@ -1436,7 +1471,7 @@ def replace_horiz_dim_debug_check(self, dim, cldicts, var_in_call_list): else: dim_string = ":" lbound_string = '1' - ubound_string = f'{udim_lname}-{ldim_lname}+1' + ubound_string = f'abs({udim_lname}-{ldim_lname})+1' return (dim_length, dim_string, lbound_string, ubound_string) def write_var_debug_check(self, var, internal_var, cldicts, outfile, errcode, errmsg, indent): @@ -1539,10 +1574,12 @@ def write_var_debug_check(self, var, internal_var, cldicts, outfile, errcode, er (ldim, udim) = dim.split(":") # Get dimension for lower bound for var_dict in cldicts: - dvar = var_dict.find_variable(standard_name=ldim, any_scope=False) + dvar = var_dict.find_variable(standard_name=ldim.lower(), any_scope=False) if dvar is not None: ldim_lname = dvar.get_prop_value('local_name') break + # end if + # end for if not dvar: # To allow for numerical dimensions in metadata. if ldim.isnumeric(): @@ -1553,10 +1590,12 @@ def write_var_debug_check(self, var, internal_var, cldicts, outfile, errcode, er # endif # Get dimension for upper bound for var_dict in cldicts: - dvar = var_dict.find_variable(standard_name=udim, any_scope=False) + dvar = var_dict.find_variable(standard_name=udim.lower(), any_scope=False) if dvar is not None: udim_lname = dvar.get_prop_value('local_name') break + # end if + # end for if not dvar: # To allow for numerical dimensions in metadata. if udim.isnumeric(): @@ -1566,7 +1605,7 @@ def write_var_debug_check(self, var, internal_var, cldicts, outfile, errcode, er # end if # end if # Assemble dimensions and bounds for size checking - dim_length = f'{udim_lname}-{ldim_lname}+1' + dim_length = f'abs({udim_lname}-{ldim_lname})+1' dim_string = ":" lbound_string = ldim_lname ubound_string = udim_lname @@ -1649,40 +1688,89 @@ def write_var_debug_check(self, var, internal_var, cldicts, outfile, errcode, er outfile.write('',indent) # endif # end if + # end if + # end def - def associate_optional_var(self, dict_var, var, var_ptr, has_transform, cldicts, indent, outfile): - """Write local pointer association for optional variables.""" + def associate_optional_var(self, dict_var, var, has_transform, cldicts, indent, outfile): + """Write local pointer association for optional variable.""" + # Need to use local_name in Group's call list (self.__group.call_list), not + # the local_name in var. + sname = var.get_prop_value('standard_name') + svar = self.__group.call_list.find_variable(standard_name=sname, any_scope=False) if (dict_var): - (conditional, _) = dict_var.conditional(cldicts) + (conditional, vars_needed) = dict_var.conditional(cldicts) if (has_transform): - lname = var.get_prop_value('local_name')+'_local' + lname = svar.get_prop_value('local_name')+'_local' else: - lname = var.get_prop_value('local_name') + lname = svar.get_prop_value('local_name') + # end if + lname_ptr = svar.get_prop_value('local_name') + '_ptr' + # Scheme has optional varaible, host has varaible defined as Conditional (Active). + if conditional != '.true.': + outfile.write(f"if {conditional} then", indent) + outfile.write(f"{lname_ptr} => {lname}", indent+1) + outfile.write(f"end if", indent) + # Scheme has optional varaible, host has varaible defined as Mandatory. + else: + outfile.write(f"{lname_ptr} => {lname}", indent+1) + # end if + # end if + # end def + + def nullify_optional_var(self, dict_var, var, has_transform, cldicts, indent, outfile): + """Write local pointer nullification for optional variable.""" + # Need to use local_name in Group's call list (self.__group.call_list), not + # the local_name in var. + sname = var.get_prop_value('standard_name') + svar = self.__group.call_list.find_variable(standard_name=sname, any_scope=False) + if (dict_var): + (conditional, vars_needed) = dict_var.conditional(cldicts) + if (has_transform): + lname = svar.get_prop_value('local_name')+'_local' + else: + lname = svar.get_prop_value('local_name') + # end if + lname_ptr = svar.get_prop_value('local_name') + '_ptr' + # Scheme has optional varaible, host has varaible defined as Conditional (Active). + if conditional != '.true.': + outfile.write(f"if {conditional} then", indent) + outfile.write(f"nullify({lname_ptr})", indent+1) + outfile.write(f"end if", indent) + # Scheme has optional varaible, host has varaible defined as Mandatory. + else: + #outfile.write(f"{lname} = {lname_ptr}", indent) + outfile.write(f"nullify({lname_ptr})", indent) # end if - lname_ptr = var_ptr.get_prop_value('local_name') - outfile.write(f"if {conditional} then", indent) - outfile.write(f"{lname_ptr} => {lname}", indent+1) - outfile.write(f"end if", indent) # end if + # end def - def assign_pointer_to_var(self, dict_var, var, var_ptr, has_transform, cldicts, indent, outfile): - """Assign local pointer to variable.""" + def assign_pointer_to_var(self, dict_var, var, has_transform, cldicts, indent, outfile): + """Write local pointer assignment to variable.""" + # Need to use local_name in Group's call list (self.__group.call_list), not + # the local_name in var. + sname = var.get_prop_value('standard_name') + svar = self.__group.call_list.find_variable(standard_name=sname, any_scope=False) if (dict_var): intent = var.get_prop_value('intent') if (intent == 'out' or intent == 'inout'): - (conditional, _) = dict_var.conditional(cldicts) + (conditional, vars_needed) = dict_var.conditional(cldicts) if (has_transform): - lname = var.get_prop_value('local_name')+'_local' + lname = svar.get_prop_value('local_name') +'_local' else: - lname = var.get_prop_value('local_name') + lname = svar.get_prop_value('local_name') + # end if + lname_ptr = svar.get_prop_value('local_name') + '_ptr' + if conditional != '.true.': + outfile.write(f"if {conditional} then", indent) + outfile.write(f"{lname} = {lname_ptr}", indent+1) + outfile.write(f"end if", indent) + else: + outfile.write(f"{lname} = {lname_ptr}", indent) # end if - lname_ptr = var_ptr.get_prop_value('local_name') - outfile.write(f"if {conditional} then", indent) - outfile.write(f"{lname} = {lname_ptr}", indent+1) - outfile.write(f"end if", indent) # end if # end if - + # end def + def add_var_transform(self, var, compat_obj, vert_dim): """Register any variable transformation needed by for this Scheme. For any transformation identified in , create dummy variable @@ -1815,7 +1903,6 @@ def write(self, outfile, errcode, errmsg, indent): subname=self.subroutine_name, sub_lname_list = self.__reverse_transforms) # - outfile.write('', indent) outfile.write('if ({} == 0) then'.format(errcode), indent) # # Write debug checks (operating on variables @@ -1825,7 +1912,6 @@ def write(self, outfile, errcode, errmsg, indent): outfile.write('! ##################################################################', indent+1) outfile.comment('Begin debug tests', indent+1) outfile.write('! ##################################################################', indent+1) - outfile.write('', indent+1) # end if for (var, internal_var) in self.__var_debug_checks: stmt = self.write_var_debug_check(var, internal_var, cldicts, outfile, errcode, errmsg, indent+1) @@ -1834,7 +1920,6 @@ def write(self, outfile, errcode, errmsg, indent): outfile.write('! ##################################################################', indent+1) outfile.comment('End debug tests', indent+1) outfile.write('! ##################################################################', indent+1) - outfile.write('', indent+1) # end if # # Write any reverse (pre-Scheme) transforms. @@ -1857,32 +1942,38 @@ def write(self, outfile, errcode, errmsg, indent): # if self.__optional_vars: outfile.write('! Associate conditional variables', indent+1) - # end if - for (dict_var, var, var_ptr, has_transform) in self.__optional_vars: - tstmt = self.associate_optional_var(dict_var, var, var_ptr, has_transform, cldicts, indent+1, outfile) + # end if + for (dict_var, var, has_transform) in self.__optional_vars: + tstmt = self.associate_optional_var(dict_var, var, has_transform, cldicts, indent+1, outfile) # end for # # Write the scheme call. # if self._has_run_phase: stmt = 'call {}({})' - outfile.write('',indent+1) outfile.write('! Call scheme', indent+1) outfile.write(stmt.format(self.subroutine_name, my_args), indent+1) - outfile.write('',indent+1) # end if # # Copy any local pointers. # first_ptr_declaration=True - for (dict_var, var, var_ptr, has_transform) in self.__optional_vars: - if first_ptr_declaration: + for (dict_var, var, has_transform) in self.__optional_vars: + if first_ptr_declaration: outfile.write('! Copy any local pointers to dummy/local variables', indent+1) first_ptr_declaration=False # end if - tstmt = self.assign_pointer_to_var(dict_var, var, var_ptr, has_transform, cldicts, indent+1, outfile) + tstmt = self.assign_pointer_to_var(dict_var, var, has_transform, cldicts, indent+1, outfile) # end for - outfile.write('',indent+1) + + # + # Nullify any local pointers. + # + if self.__optional_vars: + outfile.write('! Nullify conditional variables', indent+1) + # end if + for (dict_var, var, has_transform) in self.__optional_vars: + tstmt = self.nullify_optional_var(dict_var, var, has_transform, cldicts, indent+1, outfile) # # Write any forward (post-Scheme) transforms. # @@ -1899,7 +1990,6 @@ def write(self, outfile, errcode, errmsg, indent): lvar_lname = lvar.get_prop_value('local_name') tstmt = self.write_var_transform(lvar_lname, dummy, rindices, lindices, compat_obj, outfile, indent+1, True) # end for - outfile.write('', indent) outfile.write('end if', indent) def schemes(self): @@ -2478,7 +2568,7 @@ def write(self, outfile, host_arglist, indent, const_mod, # end if # end if # end for - # All optional dummy variables within group need to have + # All optional dummy variables within group need to have # an associated pointer array declared. for cvar in self.call_list.variable_list(): opt_var = cvar.get_prop_value('optional') @@ -2530,7 +2620,6 @@ def write(self, outfile, host_arglist, indent, const_mod, subpart_scalar_vars[lname] = (ivar, item, opt_var) # end if # end for - # end for # First, write out the subroutine header subname = self.name @@ -2548,6 +2637,9 @@ def write(self, outfile, host_arglist, indent, const_mod, for scheme in sorted(self._local_schemes): smod = scheme[0] sname = scheme[1] + # For schemes with "host variants" (e.g. GFS_time.HOST.F90) + if "." in smod: + smod = smod[0:smod.find(".")] slen = ' '*(modmax - len(smod)) outfile.write(scheme_use.format(smod, slen, sname), indent+1) # end for @@ -2598,9 +2690,13 @@ def write(self, outfile, host_arglist, indent, const_mod, allocatable=(key in optional_var_set), target=target) # end for - # Pointer variables + # Pointer variables. + outfile.write('', 0) + if pointer_var_set: + outfile.write('! Local pointer variables', indent+1) + # end if for (name, kind, dim, vtype) in pointer_var_set: - var.write_ptr_def(outfile, indent+1, name, kind, dim, vtype) + write_ptr_def(outfile, indent+1, name, kind, dim, vtype) # end for outfile.write('', 0) # Get error variable names @@ -2640,15 +2736,20 @@ def write(self, outfile, host_arglist, indent, const_mod, {'errcode' : errcode, 'errmsg' : errmsg, 'funcname' : self.name}) # Write any loop match calculations - outfile.write("! Set horizontal loop extent",indent+1) + if self._loop_var_matches: + outfile.write("! Set horizontal loop extent",indent+1) + # end if for vmatch in self._loop_var_matches: action = vmatch.write_action(self, dict2=self.call_list) if action: outfile.write(action, indent+1) # end if # end for + # end if # Allocate local arrays - outfile.write('\n! Allocate local arrays', indent+1) + if bool(allocatable_var_set): + outfile.write('\n! Allocate local arrays', indent+1) + # end if alloc_stmt = "allocate({}({}))" for lname in sorted(allocatable_var_set): var = subpart_allocate_vars[lname][0] @@ -2664,7 +2765,9 @@ def write(self, outfile, host_arglist, indent, const_mod, # end for # Allocate suite vars if allocate: - outfile.write('\n! Allocate suite_vars', indent+1) + if suite_vars.variable_list(): + outfile.write('\n! Allocate suite_vars', indent+1) + # end if for svar in suite_vars.variable_list(): dims = svar.get_dimensions() if dims: @@ -2692,14 +2795,6 @@ def write(self, outfile, host_arglist, indent, const_mod, for lname in optional_var_set: outfile.write('if (allocated({})) {} deallocate({})'.format(lname,' '*(20-len(lname)),lname), indent+1) # end for - # Nullify local pointers - if pointer_var_set: - outfile.write('\n! Nullify local pointers', indent+1) - # end if - for (name, kind, dim, vtype) in pointer_var_set: - #cspace = ' '*(15-len(name)) - outfile.write('if (associated({})) {} nullify({})'.format(name,' '*(15-len(name)),name), indent+1) - # end fo # Deallocate suite vars if deallocate: for svar in suite_vars.variable_list(): diff --git a/scripts/ufs_depends.py b/scripts/ufs_depends.py new file mode 100644 index 00000000..985fb5d7 --- /dev/null +++ b/scripts/ufs_depends.py @@ -0,0 +1,447 @@ +#!/usr/bin/env python3 +# + +# Python library imports +import sys +import os.path +import filecmp +import importlib + +# CCPP framework (prebuild) imports +from mkcap import CapsMakefile, CapsCMakefile, CapsSourcefile +from mkcap import SchemesMakefile, SchemesCMakefile, SchemesSourcefile +from mkcap import TypedefsMakefile, TypedefsCMakefile, TypedefsSourcefile + +################################################################################ +# +################################################################################ +def write_makefile(obj_, name_, name_makefile, name_cmakefile, name_sourcefile): + """Generate makefile/cmakefile snippets used by UFS and SCM""" + if obj_ == "TYPEDEFS": + makefile = TypedefsMakefile() + cmakefile = TypedefsCMakefile() + sourcefile = TypedefsSourcefile() + elif obj_ == "SCHEMES": + makefile = SchemesMakefile() + cmakefile = SchemesCMakefile() + sourcefile = SchemesSourcefile() + elif obj_ == "CAPS": + makefile = CapsMakefile() + cmakefile = CapsCMakefile() + sourcefile = CapsSourcefile() + elif obj_ == "API": + makefile = APIMakefile() + cmakefile = APICMakefile() + sourcefile = APISourcefile() + elif obj_ == "KINDS": + makefile = KindsMakefile() + cmakefile = KindsCMakefile() + sourcefile = KindsSourcefile() + else: + return + # end if + makefile.filename = name_makefile + '.tmp' + cmakefile.filename = name_cmakefile + '.tmp' + sourcefile.filename = name_sourcefile + '.tmp' + # Sort _name so that the order remains the same (for cmake to avoid) recompiling + if isinstance(name_, list): name_.sort() + # Generate list of type definitions + makefile.write(name_) + cmakefile.write(name_) + sourcefile.write(name_) + if os.path.isfile(name_makefile) and \ + filecmp.cmp(name_makefile, makefile.filename): + os.remove(makefile.filename) + os.remove(cmakefile.filename) + os.remove(sourcefile.filename) + else: + if os.path.isfile(name_makefile): + os.remove(name_makefile) + # end if + if os.path.isfile(name_cmakefile): + os.remove(name_cmakefile) + # end if + if os.path.isfile(name_sourcefile): + os.remove(name_sourcefile) + # end if + os.rename(makefile.filename, name_makefile) + os.rename(cmakefile.filename, name_cmakefile) + os.rename(sourcefile.filename, name_sourcefile) + # end if +# end def + +################################################################################ +# +################################################################################ +def import_ccpp_cfg(configfile, builddir): + """Import the CCPP configuration from a given configuration file""" + success = True + config = {} + + # Sanity. Make sure file exists. + if not os.path.isfile(configfile): + logging.error("Configuration file {0} not found".format(configfile)) + success = False + return(success, config) + # end if + + # Import the host-model specific CCPP capgen config file split into path and + # module name for import + configpath = os.path.abspath(os.path.dirname(configfile)) + configmodule = os.path.splitext(os.path.basename(configfile))[0] + sys.path.append(configpath) + ccpp_capgen_config = importlib.import_module(configmodule) + + # If the build directory for running ccpp_capgen.py is not + # specified as command line argument, use value from config + if not builddir: + builddir = os.path.join(BASEDIR, ccpp_capgen_config.DEFAULT_BUILD_DIR) + logging.info('Build directory not specified on command line, ' + \ + 'use "{}" from CCPP capgen config'.format(ccpp_capgen_config.DEFAULT_BUILD_DIR)) + # end if + + # + config['typedefs_makefile'] = ccpp_capgen_config.TYPEDEFS_MAKEFILE.format(build_dir=builddir) + config['typedefs_cmakefile'] = ccpp_capgen_config.TYPEDEFS_CMAKEFILE.format(build_dir=builddir) + config['typedefs_sourcefile'] = ccpp_capgen_config.TYPEDEFS_SOURCEFILE.format(build_dir=builddir) + config['schemes_makefile'] = ccpp_capgen_config.SCHEMES_MAKEFILE.format(build_dir=builddir) + config['schemes_cmakefile'] = ccpp_capgen_config.SCHEMES_CMAKEFILE.format(build_dir=builddir) + config['schemes_sourcefile'] = ccpp_capgen_config.SCHEMES_SOURCEFILE.format(build_dir=builddir) + config['caps_makefile'] = ccpp_capgen_config.CAPS_MAKEFILE.format(build_dir=builddir) + config['caps_cmakefile'] = ccpp_capgen_config.CAPS_CMAKEFILE.format(build_dir=builddir) + config['caps_sourcefile'] = ccpp_capgen_config.CAPS_SOURCEFILE.format(build_dir=builddir) + config['kinds_makefile'] = ccpp_capgen_config.KINDS_MAKEFILE.format(build_dir=builddir) + config['kinds_cmakefile'] = ccpp_capgen_config.KINDS_CMAKEFILE.format(build_dir=builddir) + config['kinds_sourcefile'] = ccpp_capgen_config.KINDS_SOURCEFILE.format(build_dir=builddir) + config['static_api_makefile'] = ccpp_capgen_config.STATIC_API_MAKEFILE.format(build_dir=builddir) + config['static_api_cmakefile'] = ccpp_capgen_config.STATIC_API_CMAKEFILE.format(build_dir=builddir) + config['static_api_sourcefile'] = ccpp_capgen_config.STATIC_API_SOURCEFILE.format(build_dir=builddir) + config['html_vartable_file'] = ccpp_capgen_config.HTML_VARTABLE_FILE.format(build_dir=builddir) + config['latex_vartable_file'] = ccpp_capgen_config.LATEX_VARTABLE_FILE.format(build_dir=builddir) + + return(success, config) +# end def + +################################################################################ +# +################################################################################ +def create_scm_build(run_env, scheme_ffiles, host_ffiles, scheme_depends, + host_depends, cap_filenames, host_mods, static_api, + kinds_file): + run_env.logger.info("Creating SCM/UFS build configuration") + [success, ccpp_cfg] = import_ccpp_cfg(run_env.ccpp_cfgfile, run_env.output_dir) + + write_makefile("SCHEMES", scheme_ffiles + host_ffiles + scheme_depends + host_depends, \ + ccpp_cfg['schemes_makefile'], \ + ccpp_cfg['schemes_cmakefile'], \ + ccpp_cfg['schemes_sourcefile']) + write_makefile("CAPS", cap_filenames, \ + ccpp_cfg['caps_makefile'], \ + ccpp_cfg['caps_cmakefile'], \ + ccpp_cfg['caps_sourcefile']) + write_makefile("TYPEDEFS", host_mods, \ + ccpp_cfg['typedefs_makefile'], \ + ccpp_cfg['typedefs_cmakefile'], \ + ccpp_cfg['typedefs_sourcefile']) + write_makefile("API", static_api, \ + ccpp_cfg['static_api_makefile'], \ + ccpp_cfg['static_api_cmakefile'], \ + ccpp_cfg['static_api_sourcefile']) + write_makefile("KINDS", kinds_file, \ + ccpp_cfg['kinds_makefile'], \ + ccpp_cfg['kinds_cmakefile'], \ + ccpp_cfg['kinds_sourcefile']) +# end def + +################################################################################ +# +################################################################################ +class APIMakefile(object): + + header=''' +# The CCPP static API is defined here. +# +# This file is auto-generated using ccpp_capgen.py +# at compile time, do not edit manually. +# +API =''' + + def __init__(self, **kwargs): + self._filename = 'sys.stdout' + for key, value in kwargs.items(): + setattr(self, "_"+key, value) + # end for + # end def + def write(self, api_names): + if (self.filename is not sys.stdout): + f = open(self.filename, 'w') + else: + f = sys.stdout + # end if + contents = self.header + for api_name in api_names: + contents += ' \\\n\t {0}'.format(api_name) + # end for + f.write(contents) + + if (f is not sys.stdout): + f.close() + # end if + @property + def filename(self): + '''Get the filename of write the output to.''' + return self._filename + # end def + @filename.setter + def filename(self, value): + self._filename = value + # end def +# end class + +################################################################################ +# +################################################################################ +class APICMakefile(object): + + header=''' +# The CCPP static API is defined here. +# +# This file is auto-generated using ccpp_capgen.py +# at compile time, do not edit manually. +# +''' + + def __init__(self, **kwargs): + self._filename = 'sys.stdout' + for key, value in kwargs.items(): + setattr(self, "_"+key, value) + # end for + # end def + def write(self, api_file): + if (self.filename is not sys.stdout): + f = open(self.filename, 'w') + else: + f = sys.stdout + # end if + contents = self.header + contents += """set(API \"{filename}\")""".format(filename=api_file) + f.write(contents) + + if (f is not sys.stdout): + f.close() + # end if + # end def + @property + def filename(self): + '''Get the filename of write the output to.''' + return self._filename + # end def + @filename.setter + def filename(self, value): + self._filename = value + # end def +# end class + +################################################################################ +# +################################################################################ +class APISourcefile(object): + + header=''' +# The CCPP static API is defined here. +# +# This file is auto-generated using ccpp_capgen.py +# at compile time, do not edit manually. +# +export API="''' + footer='''" +''' + + def __init__(self, **kwargs): + self._filename = 'sys.stdout' + for key, value in kwargs.items(): + setattr(self, "_"+key, value) + # end for + # end def + def write(self, api_names): + if (self.filename is not sys.stdout): + filepath = os.path.split(self.filename)[0] + if filepath and not os.path.isdir(filepath): + os.makedirs(filepath) + # end if + f = open(self.filename, 'w') + else: + f = sys.stdout + # end if + contents = self.header + for api_name in api_names: + contents += '{0};'.format(api_name) + # end for + contents = contents.rstrip(';') + contents += self.footer + f.write(contents) + + if (f is not sys.stdout): + f.close() + # end if + # end def + @property + def filename(self): + '''Get the filename of write the output to.''' + return self._filename + # end def + @filename.setter + def filename(self, value): + self._filename = value + # end def +# end class + +################################################################################ +# +################################################################################ +class KindsMakefile(object): + + header=''' +# The CCPP kinds file is defined here. +# +# This file is auto-generated using ccpp_capgen.py +# at compile time, do not edit manually. +# +KINDS =''' + + def __init__(self, **kwargs): + self._filename = 'sys.stdout' + for key, value in kwargs.items(): + setattr(self, "_"+key, value) + # end for + # end def + def write(self, kinds): + if (self.filename is not sys.stdout): + f = open(self.filename, 'w') + else: + f = sys.stdout + # end if + contents = self.header + for kind in kinds: + contents += ' \\\n\t {0}'.format(kind) + # end for + f.write(contents) + + if (f is not sys.stdout): + f.close() + # end if + # end def + @property + def filename(self): + '''Get the filename of write the output to.''' + return self._filename + # end def + @filename.setter + def filename(self, value): + self._filename = value + # end def +# end class + +################################################################################ +# +################################################################################ +class KindsCMakefile(object): + + header=''' +# All CCPP Kinds is defined here. +# +# This file is auto-generated using ccpp_capgen.py +# at compile time, do not edit manually. +# +set(KINDS +''' + footer=''') +''' + + def __init__(self, **kwargs): + self._filename = 'sys.stdout' + for key, value in kwargs.items(): + setattr(self, "_"+key, value) + # end for + # end def + def write(self, kinds): + if (self.filename is not sys.stdout): + f = open(self.filename, 'w') + else: + f = sys.stdout + # end if + contents = self.header + for kind in kinds: + contents += ' {0}\n'.format(kind) + # end for + contents += self.footer + f.write(contents) + + if (f is not sys.stdout): + f.close() + # end if + @property + def filename(self): + '''Get the filename of write the output to.''' + return self._filename + # end def + @filename.setter + def filename(self, value): + self._filename = value + # end def +# end class + +################################################################################ +# +################################################################################ +class KindsSourcefile(object): + + header=''' +# The CCPP Kinds file is defined here. +# +# This file is auto-generated using ccpp_capgen.py +# at compile time, do not edit manually. +# +export KINDS="''' + footer='''" +''' + + def __init__(self, **kwargs): + self._filename = 'sys.stdout' + for key, value in kwargs.items(): + setattr(self, "_"+key, value) + # end for + # end def + + def write(self, kinds): + if (self.filename is not sys.stdout): + filepath = os.path.split(self.filename)[0] + if filepath and not os.path.isdir(filepath): + os.makedirs(filepath) + # end if + f = open(self.filename, 'w') + else: + f = sys.stdout + # end if + + contents = self.header + for kind in kinds: + contents += '{0};'.format(kind) + # end for + contents = contents.rstrip(';') + contents += self.footer + f.write(contents) + + if (f is not sys.stdout): + f.close() + # end if + @property + def filename(self): + '''Get the filename of write the output to.''' + return self._filename + # end def + @filename.setter + def filename(self, value): + self._filename = value + # end def +# end class diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a7428a77..9c017e50 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -6,11 +6,21 @@ set(SOURCES_F90 ccpp_types.F90 ) -set(CMAKE_Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_INCLUDEDIR}) +if(CCPP_SCM) + set(SOURCES_F90 + ccpp_types.F90 + ccpp_hashable.F90 + ccpp_hash_table.F90 + ccpp_constituent_prop_mod.F90 + ccpp_scheme_utils.F90 + ) + message(STATUS "Got CCPP KINDS from cmakefile include file: ${KINDS}") +endif() +set(CMAKE_Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) #------------------------------------------------------------------------------ # Define the executable and what to link -add_library(ccpp_framework STATIC ${SOURCES_F90}) +add_library(ccpp_framework STATIC ${KINDS} ${SOURCES_F90}) target_link_libraries(ccpp_framework PUBLIC MPI::MPI_Fortran) if(OPENMP) target_link_libraries(ccpp_framework PUBLIC OpenMP::OpenMP_Fortran) diff --git a/src/ccpp_types.meta b/src/ccpp_types.meta index cdec1dc2..8211ff1c 100644 --- a/src/ccpp_types.meta +++ b/src/ccpp_types.meta @@ -89,12 +89,6 @@ units = DDT dimensions = () type = ccpp_t -[one] - standard_name = ccpp_constant_one - long_name = definition of constant one - units = 1 - dimensions = () - type = integer [MPI_Comm] standard_name = MPI_Comm long_name = definition of type MPI_Comm diff --git a/test/capgen_test/capgen_test_reports.py b/test/capgen_test/capgen_test_reports.py index 292c1a65..b8943c9f 100644 --- a/test/capgen_test/capgen_test_reports.py +++ b/test/capgen_test/capgen_test_reports.py @@ -37,9 +37,9 @@ os.path.join(_BUILD_DIR, "ccpp", "ccpp_ddt_suite_cap.F90"), os.path.join(_BUILD_DIR, "ccpp", "ccpp_temp_suite_cap.F90")] _DEPENDENCIES = [os.path.join(_TEST_DIR, "adjust", "qux.F90"), - os.path.join(_TEST_DIR, "ddt2"), - os.path.join(_TEST_DIR, "bar.F90"), - os.path.join(_TEST_DIR, "foo.F90")] + os.path.join(_TEST_DIR, "ddt2")]#, + #os.path.join(_TEST_DIR, "bar.F90"), + #os.path.join(_TEST_DIR, "foo.F90")] _PROCESS_LIST = ["setter=temp_set", "adjusting=temp_calc_adjust"] _MODULE_LIST = ["environ_conditions", "make_ddt", "setup_coeffs", "temp_adjust", "temp_calc_adjust", "temp_set"] diff --git a/test/capgen_test/temp_calc_adjust.meta b/test/capgen_test/temp_calc_adjust.meta index e014fb6e..65cde755 100644 --- a/test/capgen_test/temp_calc_adjust.meta +++ b/test/capgen_test/temp_calc_adjust.meta @@ -1,7 +1,7 @@ [ccpp-table-properties] name = temp_calc_adjust type = scheme - dependencies = foo.F90, bar.F90 + [ccpp-arg-table] name = temp_calc_adjust_register type = scheme diff --git a/test/ddthost_test/ddthost_test_reports.py b/test/ddthost_test/ddthost_test_reports.py index 612cbbbf..f5de081e 100644 --- a/test/ddthost_test/ddthost_test_reports.py +++ b/test/ddthost_test/ddthost_test_reports.py @@ -37,9 +37,9 @@ [os.path.join(_BUILD_DIR, "ccpp", "test_host_ccpp_cap.F90"), os.path.join(_BUILD_DIR, "ccpp", "ccpp_ddt_suite_cap.F90"), os.path.join(_BUILD_DIR, "ccpp", "ccpp_temp_suite_cap.F90")] -_DEPENDENCIES = [os.path.join(_TEST_DIR, "adjust", "qux.F90"), - os.path.join(_TEST_DIR, "bar.F90"), - os.path.join(_TEST_DIR, "foo.F90")] +_DEPENDENCIES = [os.path.join(_TEST_DIR, "adjust", "qux.F90")]#, +# os.path.join(_TEST_DIR, "bar.F90"), +# os.path.join(_TEST_DIR, "foo.F90")] _PROCESS_LIST = ["setter=temp_set", "adjusting=temp_calc_adjust"] _MODULE_LIST = ["environ_conditions", "make_ddt", "setup_coeffs", "temp_adjust", "temp_calc_adjust", "temp_set"] diff --git a/test/ddthost_test/temp_calc_adjust.meta b/test/ddthost_test/temp_calc_adjust.meta index 437de934..8e017d57 100644 --- a/test/ddthost_test/temp_calc_adjust.meta +++ b/test/ddthost_test/temp_calc_adjust.meta @@ -1,7 +1,6 @@ [ccpp-table-properties] name = temp_calc_adjust type = scheme - dependencies = foo.F90, bar.F90 [ccpp-arg-table] name = temp_calc_adjust_run type = scheme