From 11694e52a2c99cdd1afb3e86c08f74f51e9d9d3f Mon Sep 17 00:00:00 2001 From: Xanthron Writer Date: Tue, 20 Jun 2023 17:37:55 +0200 Subject: [PATCH 01/23] Refactored program for single command usage. Now, executing the sup programs is as simple as typing propti run . To enable this functionality, the folder structure has been modified. The library and sub programs have been relocated to their respective folders. As a result, minor changes were made to the imports. Furthermore, a setup file has been included for future program distribution. --- propti/__init__.py | 87 +++++++------------ propti/__main__.py | 5 ++ propti/lib/__init__.py | 58 +++++++++++++ propti/{ => lib}/basic_functions.py | 0 propti/{ => lib}/data_structures.py | 2 +- propti/{ => lib}/fitness_methods.py | 0 propti/{ => lib}/propti_monitor.py | 2 +- propti/{ => lib}/propti_post_processing.py | 2 +- propti/{ => lib}/propti_pre_processing.py | 2 +- propti/{ => lib}/spotpy_wrapper.py | 0 .../run/propti_analyse.py | 8 +- .../run/propti_prepare.py | 2 +- propti_run.py => propti/run/propti_run.py | 2 +- propti_sense.py => propti/run/propti_sense.py | 3 +- setup.py | 44 ++++++++++ 15 files changed, 147 insertions(+), 70 deletions(-) create mode 100644 propti/__main__.py create mode 100644 propti/lib/__init__.py rename propti/{ => lib}/basic_functions.py (100%) rename propti/{ => lib}/data_structures.py (99%) rename propti/{ => lib}/fitness_methods.py (100%) rename propti/{ => lib}/propti_monitor.py (99%) rename propti/{ => lib}/propti_post_processing.py (99%) rename propti/{ => lib}/propti_pre_processing.py (99%) rename propti/{ => lib}/spotpy_wrapper.py (100%) rename propti_analyse.py => propti/run/propti_analyse.py (99%) rename propti_prepare.py => propti/run/propti_prepare.py (99%) rename propti_run.py => propti/run/propti_run.py (98%) rename propti_sense.py => propti/run/propti_sense.py (98%) create mode 100644 setup.py diff --git a/propti/__init__.py b/propti/__init__.py index 42b767a..7e1ec99 100644 --- a/propti/__init__.py +++ b/propti/__init__.py @@ -1,58 +1,29 @@ -import logging - -######### -# LOGGING -# set up logging to file - see previous section for more details - -# get MPI rank for individual log files -import mpi4py -mpi4py.rc.recv_mprobe = False - -from mpi4py import MPI -my_rank = MPI.COMM_WORLD.Get_rank() - -logging.basicConfig(level=logging.DEBUG, - format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', - datefmt='%m-%d %H:%M', - filename='propti.{:03d}.log'.format(my_rank), - filemode='w') - -# define a Handler which writes INFO messages or higher to the sys.stderr -console = logging.StreamHandler() -console.setLevel(logging.INFO) - -# set a format which is simpler for console use -formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') - -# tell the handler to use this format -console.setFormatter(formatter) - -# add the handler to the root logger -logging.getLogger('').addHandler(console) - -######## -# PROPTI AND SPOTPY - -from .spotpy_wrapper import run_optimisation, create_input_file -from .data_structures import Parameter, ParameterSet, \ - SimulationSetupSet, SimulationSetup, Relation, DataSource, \ - OptimiserProperties, Version -from .basic_functions import run_simulations -from .propti_post_processing import run_best_para - -from .propti_monitor import plot_scatter, plot_scatter2, \ - plot_para_vs_fitness, plot_box_rmse -from .propti_post_processing import run_best_para, plot_hist, \ - calc_pearson_coefficient, collect_best_para_multi, plot_best_sim_exp -from .propti_pre_processing import interpolate_lists - -from .fitness_methods import FitnessMethodRMSE, FitnessMethodInterface, \ - FitnessMethodThreshold, FitnessMethodRangeRMSE, FitnessMethodBandRMSE, \ - FitnessMethodIntegrate - - -########### -# CONSTANTS - -# TODO: respect this variable in scripts -pickle_prefix = 'propti.pickle' +from .lib import * + +__version__ = "0.2.0" + +def main(): + import argparse + import sys + + commands = ["analyse","prepare","run","sense"] + + command = sys.argv[1] if len(sys.argv) > 1 else None + if command in commands: + sys.argv.pop(1) + if command == "analyse": + from .run import propti_analyse + elif command == "prepare": + from .run import propti_prepare + elif command == "run": + from .run import propti_run + elif command == "sense": + from .run import propti_sense + else: + parser = argparse.ArgumentParser( + prog='propti', + description='Test calling different sub programs', + epilog="use: 'propti -h' for more information about the sub program.") + parser.add_argument("command",choices=commands) + parser.add_argument("args", nargs="*") + parser.parse_args() \ No newline at end of file diff --git a/propti/__main__.py b/propti/__main__.py new file mode 100644 index 0000000..03e167f --- /dev/null +++ b/propti/__main__.py @@ -0,0 +1,5 @@ +from .lib import main + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/propti/lib/__init__.py b/propti/lib/__init__.py new file mode 100644 index 0000000..42b767a --- /dev/null +++ b/propti/lib/__init__.py @@ -0,0 +1,58 @@ +import logging + +######### +# LOGGING +# set up logging to file - see previous section for more details + +# get MPI rank for individual log files +import mpi4py +mpi4py.rc.recv_mprobe = False + +from mpi4py import MPI +my_rank = MPI.COMM_WORLD.Get_rank() + +logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', + datefmt='%m-%d %H:%M', + filename='propti.{:03d}.log'.format(my_rank), + filemode='w') + +# define a Handler which writes INFO messages or higher to the sys.stderr +console = logging.StreamHandler() +console.setLevel(logging.INFO) + +# set a format which is simpler for console use +formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') + +# tell the handler to use this format +console.setFormatter(formatter) + +# add the handler to the root logger +logging.getLogger('').addHandler(console) + +######## +# PROPTI AND SPOTPY + +from .spotpy_wrapper import run_optimisation, create_input_file +from .data_structures import Parameter, ParameterSet, \ + SimulationSetupSet, SimulationSetup, Relation, DataSource, \ + OptimiserProperties, Version +from .basic_functions import run_simulations +from .propti_post_processing import run_best_para + +from .propti_monitor import plot_scatter, plot_scatter2, \ + plot_para_vs_fitness, plot_box_rmse +from .propti_post_processing import run_best_para, plot_hist, \ + calc_pearson_coefficient, collect_best_para_multi, plot_best_sim_exp +from .propti_pre_processing import interpolate_lists + +from .fitness_methods import FitnessMethodRMSE, FitnessMethodInterface, \ + FitnessMethodThreshold, FitnessMethodRangeRMSE, FitnessMethodBandRMSE, \ + FitnessMethodIntegrate + + +########### +# CONSTANTS + +# TODO: respect this variable in scripts +pickle_prefix = 'propti.pickle' diff --git a/propti/basic_functions.py b/propti/lib/basic_functions.py similarity index 100% rename from propti/basic_functions.py rename to propti/lib/basic_functions.py diff --git a/propti/data_structures.py b/propti/lib/data_structures.py similarity index 99% rename from propti/data_structures.py rename to propti/lib/data_structures.py index 1a50783..3909f8a 100644 --- a/propti/data_structures.py +++ b/propti/lib/data_structures.py @@ -11,7 +11,7 @@ from .fitness_methods import FitnessMethodInterface from typing import Union -import propti as pr +from .. import lib as pr from typing import List diff --git a/propti/fitness_methods.py b/propti/lib/fitness_methods.py similarity index 100% rename from propti/fitness_methods.py rename to propti/lib/fitness_methods.py diff --git a/propti/propti_monitor.py b/propti/lib/propti_monitor.py similarity index 99% rename from propti/propti_monitor.py rename to propti/lib/propti_monitor.py index f127585..0d72d26 100644 --- a/propti/propti_monitor.py +++ b/propti/lib/propti_monitor.py @@ -5,7 +5,7 @@ @author: thehnen; based on a script from belt """ -import propti as pr +from .. import lib as pr import numpy as np import pandas as pd import scipy.signal as sign diff --git a/propti/propti_post_processing.py b/propti/lib/propti_post_processing.py similarity index 99% rename from propti/propti_post_processing.py rename to propti/lib/propti_post_processing.py index 53f2ddb..254ba14 100644 --- a/propti/propti_post_processing.py +++ b/propti/lib/propti_post_processing.py @@ -12,7 +12,7 @@ import logging import subprocess -import propti as pr +from .. import lib as pr import numpy as np import pandas as pd diff --git a/propti/propti_pre_processing.py b/propti/lib/propti_pre_processing.py similarity index 99% rename from propti/propti_pre_processing.py rename to propti/lib/propti_pre_processing.py index 791d721..b8a52f7 100644 --- a/propti/propti_pre_processing.py +++ b/propti/lib/propti_pre_processing.py @@ -4,7 +4,7 @@ import shutil as sh import logging -import propti as pr +from .. import lib as pr import statistics as stat import numpy as np diff --git a/propti/spotpy_wrapper.py b/propti/lib/spotpy_wrapper.py similarity index 100% rename from propti/spotpy_wrapper.py rename to propti/lib/spotpy_wrapper.py diff --git a/propti_analyse.py b/propti/run/propti_analyse.py similarity index 99% rename from propti_analyse.py rename to propti/run/propti_analyse.py index bece9a9..f715756 100644 --- a/propti_analyse.py +++ b/propti/run/propti_analyse.py @@ -7,13 +7,13 @@ import copy import shutil -import propti.basic_functions as pbf # import matplotlib.pyplot as plt -import propti as pr -import propti.propti_monitor as pm -import propti.propti_post_processing as ppm +from .. import lib as pr +from ..lib import basic_functions as pbf +from ..lib import propti_monitor as pm +from ..lib import propti_post_processing as ppm parser = argparse.ArgumentParser() diff --git a/propti_prepare.py b/propti/run/propti_prepare.py similarity index 99% rename from propti_prepare.py rename to propti/run/propti_prepare.py index 7772066..e0a872f 100644 --- a/propti_prepare.py +++ b/propti/run/propti_prepare.py @@ -6,7 +6,7 @@ import shutil as sh import pickle -import propti as pr +from .. import lib as pr import logging diff --git a/propti_run.py b/propti/run/propti_run.py similarity index 98% rename from propti_run.py rename to propti/run/propti_run.py index 2118334..24a849c 100644 --- a/propti_run.py +++ b/propti/run/propti_run.py @@ -5,7 +5,7 @@ import pandas as pd import shutil as sh import pickle -import propti as pr +from .. import lib as pr import logging import argparse diff --git a/propti_sense.py b/propti/run/propti_sense.py similarity index 98% rename from propti_sense.py rename to propti/run/propti_sense.py index 47dcde4..953b54b 100644 --- a/propti_sense.py +++ b/propti/run/propti_sense.py @@ -1,9 +1,8 @@ import numpy as np -import propti as pr import pickle -import propti as pr +from .. import lib as pr import logging diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..3e935e1 --- /dev/null +++ b/setup.py @@ -0,0 +1,44 @@ +#TODO + +import os +import setuptools +#import propti + +base_dir = os.path.dirname(os.path.realpath(__file__)) +with open(os.path.join(base_dir, "README.md"), 'r', encoding="utf-8") as f: + long_description = f.read() + +setuptools.setup( + name="propti", + version="0.2.0", + # use_incremental=True, + # setup_requires=["incremental"], + # #author= + # #author_email= + description="PROPTI is an interface tool that couples simulation models with algorithms to solve the inverse problem of material parameter estimation in a modular way. It is designed with flexibility in mind and can communicate with arbitrary algorithm libraries and simulation models. Furthermore, it provides basic means of pre- and post-processing.", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/FireDynamics/propti", + classifiers=[ + "Programming Language :: Python :: 3", + "License :: MIT", + "Operating System :: OS Independent", + ], + python_requires='>=3.0', + package_dir={"propti": "propti"}, + install_requires= + [ + "numpy", + "matplotlib", + "scipy", + "pandas", + "spotpy", + "mpi4py", + # "incremental", + ], + entry_points={ + 'console_scripts': [ + 'propti = propti:main', + ] + } +) \ No newline at end of file From e55ddcfb4b890dfc15eb64dc62415b67aadd8afe Mon Sep 17 00:00:00 2001 From: Xanthron Writer Date: Tue, 20 Jun 2023 17:46:40 +0200 Subject: [PATCH 02/23] Added some paths to ignore --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d00901b..9fd9612 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,7 @@ .idea/dictionaries/thehnen.xml .idea/modules.xml .idea/workspace.xml -.idea/propti.iml \ No newline at end of file +.idea/propti.iml +__pycache__ +build/ +propti.egg-info/ \ No newline at end of file From cdc366e5802c50c03329a957de327a6795236835 Mon Sep 17 00:00:00 2001 From: Xanthron Writer Date: Fri, 30 Jun 2023 07:47:45 +0200 Subject: [PATCH 03/23] Removed requirements.txt Removed requirements.txt since it is redundant because of setup.py. --- requirements.txt | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 8b7cc3e..0000000 --- a/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -numpy -matplotlib -scipy -pandas -spotpy -mpi4py From 2e1986b4158ace0bb5be5119147f453ef5b588cd Mon Sep 17 00:00:00 2001 From: Xanthron Writer Date: Fri, 30 Jun 2023 08:01:44 +0200 Subject: [PATCH 04/23] Ramoved __main__.py --- propti/__main__.py | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 propti/__main__.py diff --git a/propti/__main__.py b/propti/__main__.py deleted file mode 100644 index 03e167f..0000000 --- a/propti/__main__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .lib import main - - -if __name__ == "__main__": - main() \ No newline at end of file From 92eb947ff5933aed4a71ac3dab9cb9874674773d Mon Sep 17 00:00:00 2001 From: Xanthron Writer Date: Fri, 30 Jun 2023 08:02:07 +0200 Subject: [PATCH 05/23] tidied up setup.py --- setup.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index 3e935e1..92132b2 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,6 @@ -#TODO - import os import setuptools -#import propti +import propti base_dir = os.path.dirname(os.path.realpath(__file__)) with open(os.path.join(base_dir, "README.md"), 'r', encoding="utf-8") as f: @@ -10,18 +8,17 @@ setuptools.setup( name="propti", - version="0.2.0", - # use_incremental=True, - # setup_requires=["incremental"], - # #author= - # #author_email= + version=propti.__version__, + #TODO #author= + #TODO #author_email= description="PROPTI is an interface tool that couples simulation models with algorithms to solve the inverse problem of material parameter estimation in a modular way. It is designed with flexibility in mind and can communicate with arbitrary algorithm libraries and simulation models. Furthermore, it provides basic means of pre- and post-processing.", long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/FireDynamics/propti", + license="MIT License", classifiers=[ "Programming Language :: Python :: 3", - "License :: MIT", + "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], python_requires='>=3.0', @@ -34,7 +31,6 @@ "pandas", "spotpy", "mpi4py", - # "incremental", ], entry_points={ 'console_scripts': [ From 271a9669be9260a23f3966c175241a1176e5b5c1 Mon Sep 17 00:00:00 2001 From: Xanthron Writer Date: Fri, 30 Jun 2023 08:03:31 +0200 Subject: [PATCH 06/23] Added build section in readme --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 3fea1dc..08f9e8e 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,16 @@ Basic functionality for data analysis of the inverse modelling process is provid Documentation is provided in [Wiki](https://github.com/FireDynamics/propti/wiki). The folder 'examples' contains application examples tested with FDS version 6.7. +## Building the Package from Source + +In the event that you have obtained this package directly from the repository, you can build it by executing the following commands: +```bash +python setup.py +pip install --upgrade dist/propti-[version number].tar.gz +# alternatively +pip install --upgrade . +``` + ## Citation PROPTI is listed to ZENODO to get Data Object Identifiers (DOI) and allow for citations in scientific papers. You can find the necessary information here: From 6cc4bc7fbdbac90025a8306526c6b05edd438f03 Mon Sep 17 00:00:00 2001 From: Xanthron Writer Date: Fri, 30 Jun 2023 08:04:10 +0200 Subject: [PATCH 07/23] Added some things to ignore --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 9fd9612..58eef29 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,6 @@ .idea/propti.iml __pycache__ build/ -propti.egg-info/ \ No newline at end of file +propti.egg-info/ +dist/ +*.log \ No newline at end of file From 560d9ee36560845d63168aadbd3ca42db44fbae6 Mon Sep 17 00:00:00 2001 From: Xanthron Writer Date: Fri, 30 Jun 2023 08:04:25 +0200 Subject: [PATCH 08/23] Updatet Version to 1.0.0 --- propti/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/propti/__init__.py b/propti/__init__.py index 7e1ec99..c6104a4 100644 --- a/propti/__init__.py +++ b/propti/__init__.py @@ -1,6 +1,6 @@ from .lib import * -__version__ = "0.2.0" +__version__ = "1.0.0" def main(): import argparse From ff09871634dae67b28eaa0504951161f9992394f Mon Sep 17 00:00:00 2001 From: Xanthron Writer Date: Fri, 7 Jul 2023 07:56:56 +0200 Subject: [PATCH 09/23] do MPI only if mpi4py can be loaded --- propti/lib/__init__.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/propti/lib/__init__.py b/propti/lib/__init__.py index 42b767a..7ef075e 100644 --- a/propti/lib/__init__.py +++ b/propti/lib/__init__.py @@ -4,12 +4,17 @@ # LOGGING # set up logging to file - see previous section for more details -# get MPI rank for individual log files -import mpi4py -mpi4py.rc.recv_mprobe = False -from mpi4py import MPI -my_rank = MPI.COMM_WORLD.Get_rank() +try: + # get MPI rank for individual log files + import mpi4py + mpi4py.rc.recv_mprobe = False + + from mpi4py import MPI + my_rank = MPI.COMM_WORLD.Get_rank() +except Exception: + my_rank = 0 + logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', From b2688840c0a76440a7cd9a2e39839f55930692d3 Mon Sep 17 00:00:00 2001 From: Xanthron Writer Date: Fri, 7 Jul 2023 07:57:14 +0200 Subject: [PATCH 10/23] example --- examples/sampling_lhs_01/cone_template.fds | 39 ++++++++++++++++++++ examples/sampling_lhs_01/input.py | 42 ++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 examples/sampling_lhs_01/cone_template.fds create mode 100644 examples/sampling_lhs_01/input.py diff --git a/examples/sampling_lhs_01/cone_template.fds b/examples/sampling_lhs_01/cone_template.fds new file mode 100644 index 0000000..ad1997a --- /dev/null +++ b/examples/sampling_lhs_01/cone_template.fds @@ -0,0 +1,39 @@ +&HEAD CHID='#CHID#', TITLE='Example from FDS user guide' / + +&MESH IJK=3,3,3, XB=-0.15,0.15,-0.15,0.15,0.0,0.3 / + +&TIME T_END=600.0, WALL_INCREMENT=1, DT=0.05 / + +&MISC SOLID_PHASE_ONLY=.TRUE. / + +&SPEC ID='METHANE' / +&MATL ID='BLACKPMMA' + ABSORPTION_COEFFICIENT=2700.0 + N_REACTIONS=1 + A(1) = 8.5E12 + E(1) = 188000 + EMISSIVITY=#EMISSIVITY# + DENSITY=#DENSITY# + SPEC_ID='METHANE' + NU_SPEC=1.0 + HEAT_OF_REACTION=870.0 + CONDUCTIVITY = #CONDUCTIVITY# + SPECIFIC_HEAT = #SPECIFIC_HEAT# / + +&SURF ID='PMMA SLAB' + COLOR='BLACK' + BACKING='INSULATED' + MATL_ID='BLACKPMMA' + THICKNESS=0.0085 + EXTERNAL_FLUX=50 / + +&VENT XB=-0.05,0.05,-0.05,0.05,0.0,0.0, SURF_ID = 'PMMA SLAB' / + +&DUMP DT_DEVC=5.0 / + +&DEVC XYZ=0.0,0.0,0.0, IOR=3, QUANTITY='WALL TEMPERATURE', ID='temp' / + +&DEVC XYZ=0.0,0.0,0.0, IOR=3, QUANTITY='MASS FLUX', SPEC_ID='METHANE', ID='MF' / +&DEVC XYZ=0.0,0.0,0.0, IOR=3, QUANTITY='WALL THICKNESS', ID='thick' / + +&TAIL / \ No newline at end of file diff --git a/examples/sampling_lhs_01/input.py b/examples/sampling_lhs_01/input.py new file mode 100644 index 0000000..e59abe3 --- /dev/null +++ b/examples/sampling_lhs_01/input.py @@ -0,0 +1,42 @@ +# define variable 'params': sampling parameter set +# define variable 'setups': simulation setup set +# define variable 'optimiser': properties for the optimiser + +# import just for IDE convenience +import propti as pr + +# fix the chid +CHID = 'CONE' + +# define the optimisation parameter +op1 = pr.Parameter(name='density', place_holder='DENSITY', + min_value=1e2, max_value=1e4) +op2 = pr.Parameter(name='emissivity', place_holder='EMISSIVITY', + min_value=0.01, max_value=1) +op3 = pr.Parameter(name='conductivity', place_holder='CONDUCTIVITY', + min_value=0.01, max_value=1) +op4 = pr.Parameter(name='specific_heat', place_holder='SPECIFIC_HEAT', + min_value=0.01, max_value=10) +ops = pr.ParameterSet(params=[op1, op2, op3, op4]) + +# define general model parameter, including optimisation parameter +params = pr.ParameterSet(params=[op1, op2, op3, op4]) +params.append(pr.Parameter(name='chid', place_holder='CHID', value=CHID)) + +# define empty simulation setup set +setups = pr.SimulationSetupSet() + +# create simulation setup object +template_file = "cone_template.fds" +s = pr.SimulationSetup(name='cone_pmma', + work_dir='cone_pmma', + execution_dir_prefix='samples_cone', + model_template=template_file, + model_parameter=params, + relations=None) + +setups.append(s) + +# use default values for optimiser +sampler = pr.Sampler(algorithm='LHS', + nsamples=10) \ No newline at end of file From 3a2cdcd36d8292fef1f331e1abcca8fbc0c033b1 Mon Sep 17 00:00:00 2001 From: Xanthron Writer Date: Fri, 7 Jul 2023 07:59:27 +0200 Subject: [PATCH 11/23] added execution_dir_prefix to wd --- propti/lib/basic_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/propti/lib/basic_functions.py b/propti/lib/basic_functions.py index d16cd19..decdfa8 100644 --- a/propti/lib/basic_functions.py +++ b/propti/lib/basic_functions.py @@ -32,7 +32,7 @@ def create_input_file(setup: SimulationSetup, work_dir='execution'): # # small test if work_dir == 'execution': - wd = setup.execution_dir + wd = os.path.join(setup.execution_dir_prefix, setup.execution_dir) elif work_dir == 'best': wd = setup.best_dir # From bcb13da4e7c62f7d83f30a16e4b355fefc3c1e60 Mon Sep 17 00:00:00 2001 From: Xanthron Writer Date: Fri, 7 Jul 2023 08:01:03 +0200 Subject: [PATCH 12/23] Added Sampling --- propti/lib/data_structures.py | 93 ++++++++++++++++++++++++ propti/run/propti_sampling.py | 132 ++++++++++++++++++++++++++++++++++ 2 files changed, 225 insertions(+) create mode 100644 propti/run/propti_sampling.py diff --git a/propti/lib/data_structures.py b/propti/lib/data_structures.py index 3909f8a..9af5b19 100644 --- a/propti/lib/data_structures.py +++ b/propti/lib/data_structures.py @@ -1003,6 +1003,99 @@ def test_simulation_setup_setup(): print(sss) +############### +# SAMPLER CLASS +class Sampler: + """ + Stores sampler parameters and provides sampling schemes. + """ + + def __init__(self, + algorithm: str = 'LHS', + nsamples: int = 12, + deterministic: bool = False, + seed: int = None, + db_name: str = 'propti_db', + db_type: str = 'csv', + db_precision=np.float64): + """ + Constructor. + :param algorithm: choose sampling algorithm, default: LHS, + range: [LHS] + :param nsamples: number of samples, default: 12 + :param deterministic: If possible, use a deterministic sampling, + default: false + :param seed: If possible, set the seed for the random number generator, + default: None + :param db_name: name of spotpy database file, default: propti_db + :param db_type: type of database, default: csv, range: [csv] + :param db_precision: desired precision of the values to be written into + the data base, default: np.float64 + """ + self.algorithm = algorithm + self.nsamples = nsamples + self.deterministic = deterministic + self.seed = seed + self.db_name = db_name + self.db_type = db_type + self.db_precision = db_precision + + def __str__(self) -> str: + """ + Pretty print of (major) class values + :return: string + """ + + return "\nsampler properties\n" \ + "--------------------\n" \ + "alg: {}\nsamples: {}\ndeterministic: {}\nseed: {}" \ + "\ndb_name: {}\ndb_type: {}" \ + "\ndb_precision: {}\n".format(self.algorithm, + self.nsamples, + self.deterministic, + self.seed, + self.db_name, + self.db_type, + self.db_precision) + + def create_sample_set(self, params: ParameterSet) -> List[ParameterSet]: + if self.algorithm == 'LHS': + logging.info("Using LHS sampler") + import scipy.stats + + + + bounds_low = [] + bounds_high = [] + param_name = [] + for p in params: + print(p) + if p.value is None: + print( p.min_value, p.max_value) + bounds_low.append(p.min_value) + bounds_high.append(p.max_value) + param_name.append(p.name) + + sample_dim = len(bounds_low) + + sampler = scipy.stats.qmc.LatinHypercube(d = sample_dim) + sample_raw = sampler.random(self.nsamples) + sample_scaled = scipy.stats.qmc.scale(sample_raw, l_bounds=bounds_low, u_bounds=bounds_high) + + sampling_set = [] + sampling_index = 0 + for ps in sample_scaled: + new_sample = ParameterSet(name=f"sample_{sampling_index:06d}", params=params) + sampling_index += 1 + for ip in range(sample_dim): + new_sample[new_sample.get_index_by_name(param_name[ip])].value = ps[ip] + sampling_set.append(new_sample) + + return sampling_set + + logging.critical("No maching sampler algorithm found.") + + ###### # MAIN diff --git a/propti/run/propti_sampling.py b/propti/run/propti_sampling.py new file mode 100644 index 0000000..ef18888 --- /dev/null +++ b/propti/run/propti_sampling.py @@ -0,0 +1,132 @@ +import sys +import os +import typing +import numpy as np +import copy +import pandas as pd +import shutil as sh +import scipy + +import propti as pr + +import logging + +import argparse +parser = argparse.ArgumentParser() +parser.add_argument("input_file", type=str, + help="python input file containing parameters and " + "simulation setups") +parser.add_argument("--root_dir", type=str, + help="root directory for sampling process", default='.') +cmdl_args = parser.parse_args() + +setups = None # type: pr.SimulationSetupSet +params = None # type: pr.ParameterSet +sampler = None # type: pr.Sampler + +input_file = cmdl_args.input_file + +logging.info("reading input file: {}".format(input_file)) +exec(open(input_file).read(), globals()) + +# TODO: check for correct execution +if params is None: + logging.critical("sampling parameters not defined") +if setups is None: + logging.critical("simulation setups not defined") +if sampler is None: + logging.critical("sampler properties not defined") + +input_file_directory = os.path.dirname(input_file) +logging.info("input file directory: {}".format(input_file_directory)) + + +# TODO: put the following lines into a general function (basic_functions.py)? +for s in setups: + + cdir = os.path.join(cmdl_args.root_dir, s.work_dir) + + # create work directories + if not os.path.exists(cdir): + os.mkdir(cdir) + + # copy model template + sh.copy(os.path.join(input_file_directory, s.model_template), cdir) + + s.model_template = os.path.join(cdir, os.path.basename(s.model_template)) + + # TODO Sampler: add comparison capability + # # copy all experimental data + # # TODO: Re-think the copy behaviour. If file is identical, just keep one + # # instance? + # for r in s.relations: + # if r.experiment is not None: + # sh.copy(os.path.join(input_file_directory, r.experiment.file_name), cdir) + # r.experiment.file_name = os.path.join(cdir, os.path.basename(r.experiment.file_name)) + +# check for potential non-unique model input files +in_file_list = [] +for s in setups: + tpath = os.path.join(s.work_dir, s.model_input_file) + logging.debug("check if {} is in {}".format(tpath, in_file_list)) + if tpath in in_file_list: + logging.error("non unique module input file path: {}".format(tpath)) + sys.exit() + in_file_list.append(tpath) + +logging.info(setups) +logging.info(params) +logging.info(sampler) + +for s in setups: + os.mkdir(s.execution_dir_prefix) + sample_set = sampler.create_sample_set(s.model_parameter) + + para_table_file = open(os.path.join(s.execution_dir_prefix, 'sample_table.csv'), 'w') + + line_name = "# NAME - index" + line_min = f"# MIN - {0:6d}" + line_max = f"# MAX - {sampler.nsamples-1:6d}" + lines_consts = "" + next_tabs = 1 + for p in s.model_parameter: + if p.max_value is None or p.min_value is None: + lines_consts += f"# CONST - {p.name} = {p.value}\n" + continue + line_name += ','+'\t'*next_tabs + p.name + line_min += f",\t{p.min_value:.6e}" + line_max += f",\t{p.max_value:.6e}" + + next_tabs = len(p.name) // 8 + + para_table_file.write(lines_consts) + para_table_file.write(line_name + '\n') + para_table_file.write(line_min + '\n') + para_table_file.write(line_max + '\n') + + logging.debug("Sample set:\n") + sample_index = 0 + for sample in sample_set: + logging.debug(f" {sample}") + + line = f"\t\t{sample_index:6d}" + for p in sample: + if p.max_value is None or p.min_value is None: + continue + line += ',\t' + f"{p.value:.6e}" + para_table_file.write(line + '\n') + + tmp_simulation_setup = typing.cast(pr.SimulationSetup, copy.deepcopy(s)) + tmp_simulation_setup.model_parameter = sample + tmp_simulation_setup.execution_dir = sample.name + os.mkdir(os.path.join(tmp_simulation_setup.execution_dir_prefix, tmp_simulation_setup.execution_dir)) + pr.create_input_file(tmp_simulation_setup) + + sample_index += 1 + + para_table_file.close() + +# if cmdl_args.prepare_init_inputs: +# logging.info("prepare input files with initial values") +# for s in setups: +# pr.create_input_file(s) \ No newline at end of file From b3e4c52155f51ea464cfee3ce197b6969a14550a Mon Sep 17 00:00:00 2001 From: Xanthron Writer Date: Fri, 7 Jul 2023 08:04:24 +0200 Subject: [PATCH 13/23] run sampler from commandline --- propti/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/propti/__init__.py b/propti/__init__.py index c6104a4..3100a43 100644 --- a/propti/__init__.py +++ b/propti/__init__.py @@ -6,7 +6,7 @@ def main(): import argparse import sys - commands = ["analyse","prepare","run","sense"] + commands = ["analyse","prepare","run","sampler","sense"] command = sys.argv[1] if len(sys.argv) > 1 else None if command in commands: @@ -17,6 +17,8 @@ def main(): from .run import propti_prepare elif command == "run": from .run import propti_run + elif command == "sampler": + from .run import propti_sampler elif command == "sense": from .run import propti_sense else: From 4aa46fa9a014fbb57a7afc09ba9159542ef30c02 Mon Sep 17 00:00:00 2001 From: Xanthron Writer Date: Tue, 1 Aug 2023 11:05:03 +0200 Subject: [PATCH 14/23] Fixed description for installing the package from sours --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 08f9e8e..a4c7b85 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Documentation is provided in [Wiki](https://github.com/FireDynamics/propti/wiki) In the event that you have obtained this package directly from the repository, you can build it by executing the following commands: ```bash -python setup.py +python setup.py sdist pip install --upgrade dist/propti-[version number].tar.gz # alternatively pip install --upgrade . From 7e2646d756dcb885822a35f73eae2daba2136c80 Mon Sep 17 00:00:00 2001 From: Xanthron Writer Date: Tue, 1 Aug 2023 11:09:09 +0200 Subject: [PATCH 15/23] Separated run from library loading. Moved main() in __init__ to __main__ so the whole library does not have to be loaded. This improves performances. --- propti/__init__.py | 33 +++------------------------------ propti/__main__.py | 28 ++++++++++++++++++++++++++++ run.py | 2 ++ 3 files changed, 33 insertions(+), 30 deletions(-) create mode 100644 propti/__main__.py create mode 100644 run.py diff --git a/propti/__init__.py b/propti/__init__.py index 3100a43..e7e4997 100644 --- a/propti/__init__.py +++ b/propti/__init__.py @@ -1,31 +1,4 @@ -from .lib import * +__version__ = "1.0.10" +# __version__ must be the fist line in order for setup.py to read the version. -__version__ = "1.0.0" - -def main(): - import argparse - import sys - - commands = ["analyse","prepare","run","sampler","sense"] - - command = sys.argv[1] if len(sys.argv) > 1 else None - if command in commands: - sys.argv.pop(1) - if command == "analyse": - from .run import propti_analyse - elif command == "prepare": - from .run import propti_prepare - elif command == "run": - from .run import propti_run - elif command == "sampler": - from .run import propti_sampler - elif command == "sense": - from .run import propti_sense - else: - parser = argparse.ArgumentParser( - prog='propti', - description='Test calling different sub programs', - epilog="use: 'propti -h' for more information about the sub program.") - parser.add_argument("command",choices=commands) - parser.add_argument("args", nargs="*") - parser.parse_args() \ No newline at end of file +from .lib import * \ No newline at end of file diff --git a/propti/__main__.py b/propti/__main__.py new file mode 100644 index 0000000..d5bbbda --- /dev/null +++ b/propti/__main__.py @@ -0,0 +1,28 @@ +def main(): + import argparse + import sys + commands = ["analyse","prepare","run","sampler","sense","sjob"] + + command = sys.argv[1] if len(sys.argv) > 1 else None + if command in commands: + sys.argv.pop(1) + if command == "analyse": + from .run import propti_analyse + elif command == "prepare": + from .run import propti_prepare + elif command == "run": + from .run import propti_run + elif command == "sampler": + from .run import propti_sampling + elif command == "sense": + from .run import propti_sense + elif command == "sjob": + from .run import propti_sjob + else: + parser = argparse.ArgumentParser( + prog='propti', + description='modelling (or optimisation) of parameters in computer simulation with focus on handling the communication between simulation software and optimisation algorithms', + epilog="use: 'propti -h' for more information about the sub program.") + parser.add_argument("command",choices=commands) + parser.add_argument("args", nargs="*") + parser.parse_args() \ No newline at end of file diff --git a/run.py b/run.py new file mode 100644 index 0000000..5206c12 --- /dev/null +++ b/run.py @@ -0,0 +1,2 @@ +from propti.__main__ import main +main() \ No newline at end of file From b03fec6917a0ddc5e8714d85c780dbc7b301b547 Mon Sep 17 00:00:00 2001 From: Xanthron Writer Date: Tue, 1 Aug 2023 11:12:22 +0200 Subject: [PATCH 16/23] Made mpi4py optional --- propti/lib/fitness_methods.py | 6 +++++- propti/run/propti_run.py | 14 ++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/propti/lib/fitness_methods.py b/propti/lib/fitness_methods.py index efc5d33..412cbc0 100644 --- a/propti/lib/fitness_methods.py +++ b/propti/lib/fitness_methods.py @@ -2,7 +2,11 @@ import logging import numpy as np -from mpi4py import MPI +try: + from mpi4py import MPI +except: + logging.warning("mpi4py is not installed") + pass class FitnessMethodInterface: diff --git a/propti/run/propti_run.py b/propti/run/propti_run.py index 24a849c..9f44ea9 100644 --- a/propti/run/propti_run.py +++ b/propti/run/propti_run.py @@ -9,13 +9,15 @@ import logging import argparse -import mpi4py -mpi4py.rc.recv_mprobe = False +try: + import mpi4py + mpi4py.rc.recv_mprobe = False -from mpi4py import MPI -comm = MPI.COMM_WORLD -print('Starting PROPTI on MPI rank {} out of {} ranks.'.format(comm.Get_rank(), - comm.Get_size())) + from mpi4py import MPI + comm = MPI.COMM_WORLD + print('Starting PROPTI on MPI rank {} out of {} ranks.'.format(comm.Get_rank(), comm.Get_size())) +except: + logging.warning("mpi4py is not installed") parser = argparse.ArgumentParser() From e241a2069e655b785337577095dcaaa481e3e3c2 Mon Sep 17 00:00:00 2001 From: Xanthron Writer Date: Tue, 1 Aug 2023 11:17:10 +0200 Subject: [PATCH 17/23] Moved the save path generation to separate function --- propti/lib/basic_functions.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/propti/lib/basic_functions.py b/propti/lib/basic_functions.py index decdfa8..2dda53a 100644 --- a/propti/lib/basic_functions.py +++ b/propti/lib/basic_functions.py @@ -17,9 +17,7 @@ ##################### # INPUT FILE HANDLING - -def create_input_file(setup: SimulationSetup, work_dir='execution'): - +def get_save_path(setup: SimulationSetup, work_dir='execution'): """ :param setup: specification of SimulationSetup on which to base the @@ -27,19 +25,29 @@ def create_input_file(setup: SimulationSetup, work_dir='execution'): :param work_dir: flag to indicate if the regular execution of the function (in the sense of inverse modeling) is wanted or if only a simulation of the best parameter set is desired, range:['execution', 'best'] - :return: Saves a file that is read by the simulation software as input file + :return: save path of the simulation """ - # - # small test if work_dir == 'execution': wd = os.path.join(setup.execution_dir_prefix, setup.execution_dir) elif work_dir == 'best': wd = setup.best_dir - # - # + return os.path.join(wd, setup.model_input_file) + +def create_input_file(setup: SimulationSetup, work_dir='execution'): + """ + + :param setup: specification of SimulationSetup on which to base the + simulation run + :param work_dir: flag to indicate if the regular execution of the function + (in the sense of inverse modeling) is wanted or if only a simulation + of the best parameter set is desired, range:['execution', 'best'] + :return: Saves a file that is read by the simulation software as input file + """ + + save_path = get_save_path(setup, work_dir) # Log the set working directory - logging.debug(wd) + logging.debug(save_path) in_fn = setup.model_template template_content = read_template(in_fn) @@ -51,9 +59,7 @@ def create_input_file(setup: SimulationSetup, work_dir='execution'): logging.debug(input_content) - out_fn = os.path.join(wd, setup.model_input_file) - - write_input_file(input_content, out_fn) + write_input_file(input_content, save_path) def write_input_file(content: str, filename: os.path): From cc20d3cb58e72bd1e00d7153706f72904a3d95cc Mon Sep 17 00:00:00 2001 From: Xanthron Writer Date: Tue, 1 Aug 2023 11:18:13 +0200 Subject: [PATCH 18/23] Improved sampler --- propti/lib/data_structures.py | 25 ++++++++++++++++--- propti/run/propti_sampling.py | 45 +++++++++++++++++++---------------- 2 files changed, 47 insertions(+), 23 deletions(-) diff --git a/propti/lib/data_structures.py b/propti/lib/data_structures.py index 9af5b19..6181b0b 100644 --- a/propti/lib/data_structures.py +++ b/propti/lib/data_structures.py @@ -1021,7 +1021,7 @@ def __init__(self, """ Constructor. :param algorithm: choose sampling algorithm, default: LHS, - range: [LHS] + range: [LHS,LINEAR] :param nsamples: number of samples, default: 12 :param deterministic: If possible, use a deterministic sampling, default: false @@ -1069,13 +1069,13 @@ def create_sample_set(self, params: ParameterSet) -> List[ParameterSet]: bounds_high = [] param_name = [] for p in params: - print(p) if p.value is None: print( p.min_value, p.max_value) bounds_low.append(p.min_value) bounds_high.append(p.max_value) param_name.append(p.name) + sample_dim = len(bounds_low) sampler = scipy.stats.qmc.LatinHypercube(d = sample_dim) @@ -1086,11 +1086,30 @@ def create_sample_set(self, params: ParameterSet) -> List[ParameterSet]: sampling_index = 0 for ps in sample_scaled: new_sample = ParameterSet(name=f"sample_{sampling_index:06d}", params=params) - sampling_index += 1 for ip in range(sample_dim): new_sample[new_sample.get_index_by_name(param_name[ip])].value = ps[ip] + sampling_set.append(new_sample) + sampling_index += 1 + + return sampling_set + if self.algorithm == "LINEAR": + logging.info("Using LINEAR sampler") + n = self.nsamples + sampling_set = [] + sampling_index = 0 + for i in range(n): + if n > 1: + f = i/(n-1) + else: + f = 0 + new_sample = ParameterSet(name=f"sample_{sampling_index:06d}", params=params) + for p in new_sample: + if p.value == None: + p.value = p.min_value + (p.max_value - p.min_value) * f + sampling_set.append(new_sample) + sampling_index += 1 return sampling_set logging.critical("No maching sampler algorithm found.") diff --git a/propti/run/propti_sampling.py b/propti/run/propti_sampling.py index ef18888..d77da2e 100644 --- a/propti/run/propti_sampling.py +++ b/propti/run/propti_sampling.py @@ -7,11 +7,13 @@ import shutil as sh import scipy -import propti as pr +from .. import lib as pr import logging import argparse + + parser = argparse.ArgumentParser() parser.add_argument("input_file", type=str, help="python input file containing parameters and " @@ -20,9 +22,10 @@ help="root directory for sampling process", default='.') cmdl_args = parser.parse_args() -setups = None # type: pr.SimulationSetupSet -params = None # type: pr.ParameterSet -sampler = None # type: pr.Sampler +setups: pr.SimulationSetupSet = None +params: pr.ParameterSet = None +sampler: pr.Sampler = None + input_file = cmdl_args.input_file @@ -42,19 +45,19 @@ # TODO: put the following lines into a general function (basic_functions.py)? -for s in setups: +for simulation in setups: - cdir = os.path.join(cmdl_args.root_dir, s.work_dir) + cdir = os.path.join(cmdl_args.root_dir, simulation.work_dir) # create work directories if not os.path.exists(cdir): os.mkdir(cdir) # copy model template - sh.copy(os.path.join(input_file_directory, s.model_template), cdir) - - s.model_template = os.path.join(cdir, os.path.basename(s.model_template)) + sh.copy(os.path.join(input_file_directory, simulation.model_template), cdir) + simulation.model_template = os.path.join(cdir, os.path.basename(simulation.model_template)) + simulation.model_input_file = os.path.basename(simulation.model_template) # TODO Sampler: add comparison capability # # copy all experimental data # # TODO: Re-think the copy behaviour. If file is identical, just keep one @@ -66,8 +69,8 @@ # check for potential non-unique model input files in_file_list = [] -for s in setups: - tpath = os.path.join(s.work_dir, s.model_input_file) +for simulation in setups: + tpath = os.path.join(simulation.work_dir, simulation.model_input_file) logging.debug("check if {} is in {}".format(tpath, in_file_list)) if tpath in in_file_list: logging.error("non unique module input file path: {}".format(tpath)) @@ -78,18 +81,19 @@ logging.info(params) logging.info(sampler) -for s in setups: - os.mkdir(s.execution_dir_prefix) - sample_set = sampler.create_sample_set(s.model_parameter) +for simulation in setups: + os.makedirs(simulation.execution_dir_prefix,exist_ok=True) + sample_set = sampler.create_sample_set(simulation.model_parameter) - para_table_file = open(os.path.join(s.execution_dir_prefix, 'sample_table.csv'), 'w') + para_table_file = open(os.path.join(simulation.execution_dir_prefix, 'sample_table.csv'), 'w') + para_table_file.write("#file_name = {}\n".format(os.path.basename(simulation.model_template))) line_name = "# NAME - index" line_min = f"# MIN - {0:6d}" line_max = f"# MAX - {sampler.nsamples-1:6d}" lines_consts = "" next_tabs = 1 - for p in s.model_parameter: + for p in simulation.model_parameter: if p.max_value is None or p.min_value is None: lines_consts += f"# CONST - {p.name} = {p.value}\n" continue @@ -109,17 +113,18 @@ for sample in sample_set: logging.debug(f" {sample}") - line = f"\t\t{sample_index:6d}" + line = f"\t\t{sample_index:06d}" for p in sample: if p.max_value is None or p.min_value is None: continue - line += ',\t' + f"{p.value:.6e}" + else: + line += ',\t' + f"{p.value:.6e}" para_table_file.write(line + '\n') - tmp_simulation_setup = typing.cast(pr.SimulationSetup, copy.deepcopy(s)) + tmp_simulation_setup = typing.cast(pr.SimulationSetup, copy.deepcopy(simulation)) tmp_simulation_setup.model_parameter = sample tmp_simulation_setup.execution_dir = sample.name - os.mkdir(os.path.join(tmp_simulation_setup.execution_dir_prefix, tmp_simulation_setup.execution_dir)) + os.makedirs(os.path.join(tmp_simulation_setup.execution_dir_prefix, tmp_simulation_setup.execution_dir), exist_ok=True) pr.create_input_file(tmp_simulation_setup) sample_index += 1 From 70ed191326b24b144c2d0484fd064ff7554d3cb1 Mon Sep 17 00:00:00 2001 From: Xanthron Writer Date: Tue, 1 Aug 2023 11:18:28 +0200 Subject: [PATCH 19/23] Added job creation for the sampler Create, start, cancel and monitor jobs created with the sampler with ease. It currently only works for Slurm jobs. --- propti/run/propti_sjob.py | 264 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 264 insertions(+) create mode 100644 propti/run/propti_sjob.py diff --git a/propti/run/propti_sjob.py b/propti/run/propti_sjob.py new file mode 100644 index 0000000..3bfef00 --- /dev/null +++ b/propti/run/propti_sjob.py @@ -0,0 +1,264 @@ +import argparse +import subprocess +import os +import re + +parser = argparse.ArgumentParser(add_help="Create, start and monitor sampler jobs for Slurm.") +subparser = parser.add_subparsers() +create_parser = subparser.add_parser("create", help="create the job files inside the sampler folders") +create_parser.add_argument("--time", default="0-02:00:00", help="the maximal time the job can run in the following format: days-hours:minutes:seconds"); +create_parser.set_defaults(mode="create") +start_parser = subparser.add_parser("start", help="start every sampler job (note: the create command had to be run)") +start_parser.set_defaults(mode="start") +start_parser = subparser.add_parser("cancel", help="cancel every running job for this sampler") +start_parser.set_defaults(mode="cancel") +start_parser = subparser.add_parser("info", help="start jobs") +start_parser.set_defaults(mode="info") +parser.add_argument("root_dir", type=str, + help="the folder wich contains all sampler folders (note: the folder name usually named like 'sample_[...]' )", default='.') +cmdl_args = parser.parse_args() + +root_dir = cmdl_args.root_dir + +def get_table_data(): + """ + Opens the CSV table created by the sampler, reads its contents, and extracts relevant data + such as file_name, name, and a list of ids. + + Returns: + tuple: A tuple containing three elements - file_name (str), name (str), and ids (list[str]). + - file_name: The name of the original file (with extension). + - name: The CHID name for the simulations. + - ids: The ids of every simulation + """ + sample_table_path = os.path.join(cmdl_args.root_dir, "sample_table.csv") + sample_table_file = open(sample_table_path) + + file_name = sample_table_file.readline()[13:-1] + name = file_name.replace(".fds","") + search = re.search("chid\s*=\s*(.*)",sample_table_file.readline()) + if search: + name = search.group(1) + ids = [] + for line in sample_table_file: + if line.startswith("#") or line == "": + continue + else: + ids.append(line.split(",")[0].strip()) + sample_table_file.close() + return file_name,name, ids + + +# Reading a complete file when only the last line is important is inefficient, +# therefore a helper function is created that reads a file in reverse. +# source: https://stackoverflow.com/questions/2301789/how-to-read-a-file-in-reverse-order +def reverse_readline(filename, buf_size=8192): + """ + A generator that returns the lines of a file in reverse order + """ + with open(filename, 'rb') as fh: + segment = None + offset = 0 + fh.seek(0, os.SEEK_END) + file_size = remaining_size = fh.tell() + while remaining_size > 0: + offset = min(file_size, offset + buf_size) + fh.seek(file_size - offset) + buffer = fh.read(min(remaining_size, buf_size)).decode(encoding='utf-8') + remaining_size -= buf_size + lines = buffer.split('\n') + # The first line of the buffer is probably not a complete line so + # we'll save it and append it to the last line of the next buffer + # we read + if segment is not None: + # If the previous chunk starts right from the beginning of line + # do not concat the segment to the last line of new chunk. + # Instead, yield the segment first + if buffer[-1] != '\n': + lines[-1] += segment + else: + yield segment + segment = lines[0] + for index in range(len(lines) - 1, 0, -1): + if lines[index]: + yield lines[index] + # Don't yield None if the file was empty + if segment is not None: + yield segment + + + +if cmdl_args.mode == "create": + # The The execution script for the simulation. Variables padded with # get replaced by the corresponding value, for the specific simulation. + job_file_content = """#!/bin/bash +#!/bin/sh +# Name of the Job +#SBATCH --job-name=#NAME# + +# On witch device the simulation is run +#SBATCH --partition=normal + +# Maximum time the job can run: days-hours:minutes:seconds +#SBATCH --time=#TIME# + +# Number of cores +#SBATCH --tasks-per-node=1 +#SBATCH --cpus-per-task=1 +#SBATCH --nodes=#NODES# + +# Output file name +#SBATCH --output=stdout.%j +#SBATCH --error=stderr.%j + + +cd #PATH# + +mkdir ./results +cd ./results + +# define FDS input file +FDSSTEM=../ + +# grep the CHID (used for stop file) +CHID=`sed -n "s/^.*CHID='\\([-0-9a-zA-Z_]*\\)'.*$/\\1/p" < $FDSSTEM*.fds` + +# append the start time to file 'time_start' +echo "$SLURM_JOB_ID -- `date`" >> time_start + +# handle the signal sent before the end of the wall clock time +function handler_sigusr1 +{ + # protocol stopping time + echo "$SLURM_JOB_ID -- `date`" >> time_stop + echo "`date` Shell received stop signal" + + # create FDS stop file + touch $CHID.stop + + # as manual stop was triggered, the end of simulation time was + # not reached, remove flag file 'simulation_time_end' + rm simulation_time_end + wait +} + +# register the function 'hander_sigusr1' to handle the signal send out +# just before the end of the wall clock time +trap "handler_sigusr1" SIGUSR1 + +# check for the simulation finished flag file 'simulation_time_end' +# if it is found, just quit +if [ -e simulation_time_end ]; then + ## simulation has already finished, nothing left to do + echo "FDS simulation already finished" + exit 0 +fi + +# simulation not finished yet +# create flag file to check for reaching simulation end time +touch simulation_time_end + +# Load FDS +module use -a /beegfs/larnold/modules +module load FDS + +# set the number of OMP threads +export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} + +# run FDS executable +mpiexec fds $FDSSTEM*.fds & wait + +# TODO make this more robust +# set RESTART to TRUE in FDS input file +sed -i 's/RESTART\s*=\s*(F|.FALSE.)/RESTART=.TRUE./g' $FDSSTEM*.fds + +# remove the stop file, otherwise the following chain parts +# would not run +rm $CHID.stop + """ + + + fds_file_name, name, ids = get_table_data() + for id in ids: + directory =os.path.join(root_dir,f"sample_{id}") + fds_file_path = os.path.join(directory, fds_file_name) + job_file_path = os.path.join(directory, "job.sh") + processes = 1 + fds_file = open(fds_file_path) + for line in fds_file: + line = line.strip() + if line.startswith("&MESH"): + search = re.search("MPI_PROCESS\s*=\s*([0-9]+)", line) + if search: + processes = int(search.group(1)) + 1 + process_name = f"{name}_{id}" + fds_file.close() + job_file = open(job_file_path, "w") + job = job_file_content.replace("#NAME#", process_name).replace("#TIME#", cmdl_args.time).replace("#NODES#", str(processes)).replace("#PATH#", f"\"{directory}\"") + job_file.write(job) + job_file.close() + +elif cmdl_args.mode == "start": + fds_file_name, name, ids = get_table_data() + for id in ids: + job_file_path = os.path.abspath(os.path.join(root_dir,f"sample_{id}", "job.sh")) + output = subprocess.check_output(["sbatch", job_file_path]) +elif cmdl_args.mode == "cancel": + output = subprocess.check_output(["squeue", "-o", r"\"%o;%i\""]) + root_path = str.encode("\\\"" + os.path.abspath("")) + print(root_path) + for line in output.splitlines(): + if line.startswith(root_path): + line = line[2:-3].decode("utf-8") + (_, id) = line.split(";") + subprocess.check_output(["scancel", id]) + +elif cmdl_args.mode == "info": + def text(t, c): + l = len(t) + if l > c: + return t[l - c:] + else: + return " " * (c - l) + t + + paths = [] + fds_file_name, name, ids = get_table_data() + for id in ids: + job_file_path = os.path.abspath(os.path.join(root_dir,f"sample_{id}", "job.sh")) + paths.append((job_file_path, id)) + total = len(paths) + + output = subprocess.check_output(["squeue", "-o", r"\"%o;%i;%j;%t;%M;%l\""]) + + root_path = str.encode("\\\"" + os.path.abspath("")) + print(root_path) + id, job_name, status, time, time_limit = "JOBID","NAME", "ST", "TIME", "TIME_LIMIT" + print(f"{text(job_name,30)} | {text(id,8)} | {text(status,2)} | {text(time,10)} | {text(time_limit,10)}") + for line in output.splitlines(): + if line.startswith(root_path): + line = line[2:-3].decode("utf-8") + (job_path, id, job_name, status, time, time_limit) = line.split(";") + print(f"{text(job_name,30)} | {text(id,8)} | {text(status,2)} | {text(time,10)} | {text(time_limit,10)}") + for i in range(len(paths)): + if paths[i][0] == job_path: + paths.pop(i) + break + + error = 0 + for (path, id) in paths: + info_path = os.path.join(os.path.dirname(path),"results",f"{name}.out") + if os.path.exists(info_path): + reader = reverse_readline(info_path) + line = next(reader, "") + if "FDS completed successfully" in line: + job_name, status = name + "_" + id, "F" + else: + job_name, status = name + "_" + id, "E " # Indent for easier spotting + error += 1 + else: + job_name, status = name + "_" + id, "E " # Indent for easier spotting + error += 1 + print(f"{text(job_name,30)} | -------- | {text(status,2)} | ---------- | ----------") + if error == 1: + print(f"{len(paths)}/{total} finished with {error} error.") + else: + print(f"{len(paths)}/{total} finished with {error} errors.") \ No newline at end of file From ad6d920dbd53850b16a50e59e37afe8e77cb755f Mon Sep 17 00:00:00 2001 From: Xanthron Writer Date: Tue, 1 Aug 2023 11:20:33 +0200 Subject: [PATCH 20/23] Added sampler structs to __init__ --- propti/lib/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/propti/lib/__init__.py b/propti/lib/__init__.py index 7ef075e..66f59d7 100644 --- a/propti/lib/__init__.py +++ b/propti/lib/__init__.py @@ -41,8 +41,8 @@ from .spotpy_wrapper import run_optimisation, create_input_file from .data_structures import Parameter, ParameterSet, \ SimulationSetupSet, SimulationSetup, Relation, DataSource, \ - OptimiserProperties, Version -from .basic_functions import run_simulations + OptimiserProperties, Version, Sampler +from .basic_functions import run_simulations, get_save_path from .propti_post_processing import run_best_para from .propti_monitor import plot_scatter, plot_scatter2, \ From 607142a56e1f9fbb0a242c1e9db51ac2a164b293 Mon Sep 17 00:00:00 2001 From: Xanthron Writer Date: Tue, 1 Aug 2023 11:20:50 +0200 Subject: [PATCH 21/23] Updated Setup - Setup now reads the __init__ file to retrieve the version number. So the the library does not get loaded and ask for packages that may not have been installed yet. - The package can now be installed properly. --- setup.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 92132b2..8e6ab05 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,15 @@ import os import setuptools -import propti base_dir = os.path.dirname(os.path.realpath(__file__)) with open(os.path.join(base_dir, "README.md"), 'r', encoding="utf-8") as f: long_description = f.read() +with open(os.path.join(base_dir, "propti", "__init__.py"), 'r', encoding="utf-8") as f: + version = f.readline().split("=")[-1].strip()[1:-1] setuptools.setup( name="propti", - version=propti.__version__, + version=version, #TODO #author= #TODO #author_email= description="PROPTI is an interface tool that couples simulation models with algorithms to solve the inverse problem of material parameter estimation in a modular way. It is designed with flexibility in mind and can communicate with arbitrary algorithm libraries and simulation models. Furthermore, it provides basic means of pre- and post-processing.", @@ -22,7 +23,8 @@ "Operating System :: OS Independent", ], python_requires='>=3.0', - package_dir={"propti": "propti"}, + #package_dir={"propti": "propti/"}, + packages = ["propti", "propti/lib", "propti/run"], install_requires= [ "numpy", @@ -30,11 +32,11 @@ "scipy", "pandas", "spotpy", - "mpi4py", + #"mpi4py", This should be optional since it is not possible to install on every system with out root ], entry_points={ 'console_scripts': [ - 'propti = propti:main', + 'propti = propti.__main__:main', ] } ) \ No newline at end of file From 71065a3dd4e94e4c7fe1f1b50c1add32ca3e47fa Mon Sep 17 00:00:00 2001 From: Xanthron Writer Date: Mon, 28 Aug 2023 13:04:14 +0200 Subject: [PATCH 22/23] Final changes to the sampler and job system rework of the job system by making it more modular so it is easier to implement a different scheduler. --- propti/__init__.py | 2 +- propti/__main__.py | 6 +- propti/jobs/fds | 83 +++++++++++ propti/lib/__init__.py | 2 +- propti/lib/data_structures.py | 127 ++++++++++++++-- propti/run/propti_job.py | 207 ++++++++++++++++++++++++++ propti/run/propti_sampling.py | 26 +++- propti/run/propti_sjob.py | 264 ---------------------------------- setup.py | 17 ++- 9 files changed, 447 insertions(+), 287 deletions(-) create mode 100644 propti/jobs/fds create mode 100644 propti/run/propti_job.py delete mode 100644 propti/run/propti_sjob.py diff --git a/propti/__init__.py b/propti/__init__.py index e7e4997..f027dd8 100644 --- a/propti/__init__.py +++ b/propti/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.0.10" +__version__ = "1.2.0" # __version__ must be the fist line in order for setup.py to read the version. from .lib import * \ No newline at end of file diff --git a/propti/__main__.py b/propti/__main__.py index d5bbbda..7ad05af 100644 --- a/propti/__main__.py +++ b/propti/__main__.py @@ -1,7 +1,7 @@ def main(): import argparse import sys - commands = ["analyse","prepare","run","sampler","sense","sjob"] + commands = ["analyse","prepare","run","sampler","sense","job"] command = sys.argv[1] if len(sys.argv) > 1 else None if command in commands: @@ -16,8 +16,8 @@ def main(): from .run import propti_sampling elif command == "sense": from .run import propti_sense - elif command == "sjob": - from .run import propti_sjob + elif command == "job": + from .run import propti_job else: parser = argparse.ArgumentParser( prog='propti', diff --git a/propti/jobs/fds b/propti/jobs/fds new file mode 100644 index 0000000..02113d1 --- /dev/null +++ b/propti/jobs/fds @@ -0,0 +1,83 @@ +#!/bin/bash +#!/bin/sh +# Name of the Job +#SBATCH --job-name=#CHID#_#ID# + +# On witch device the simulation is run +#SBATCH --partition=normal + +# Maximum time the job can run: days-hours:minutes:seconds +#SBATCH --time=#TIME# + +# Number of cores +#SBATCH --tasks-per-node=1 +#SBATCH --cpus-per-task=1 +#SBATCH --nodes=#NODES# + +# Output file name +#SBATCH --output=stdout.%j +#SBATCH --error=stderr.%j + + +cd #EXECUTION_DIRS# + +mkdir ./results +cd ./results + +# define FDS input file +FDSSTEM=../ + +# grep the CHID (used for stop file) +CHID=`sed -n "s/^.*CHID='\\([-0-9a-zA-Z_]*\\)'.*$/\\1/p" < $FDSSTEM*.fds` + +# append the start time to file 'time_start' +echo "$SLURM_JOB_ID -- `date`" >> time_start + +# handle the signal sent before the end of the wall clock time +function handler_sigusr1 +{ + # protocol stopping time + echo "$SLURM_JOB_ID -- `date`" >> time_stop + echo "`date` Shell received stop signal" + + # create FDS stop file + touch $CHID.stop + + # as manual stop was triggered, the end of simulation time was + # not reached, remove flag file 'simulation_time_end' + rm simulation_time_end + wait +} + +# register the function 'hander_sigusr1' to handle the signal send out +# just before the end of the wall clock time +trap "handler_sigusr1" SIGUSR1 + +# check for the simulation finished flag file 'simulation_time_end' +# if it is found, just quit +if [ -e simulation_time_end ]; then + ## simulation has already finished, nothing left to do + echo "FDS simulation already finished" + exit 0 +fi + +# simulation not finished yet +# create flag file to check for reaching simulation end time +touch simulation_time_end + +# Load FDS +module use -a /beegfs/larnold/modules +module load FDS + +# set the number of OMP threads +export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} + +# run FDS executable +mpiexec fds $FDSSTEM*.fds & wait + +# set RESTART to TRUE in FDS input file +sed -i 's/RESTART\s*=\s*\(F\|.FALSE.\)/RESTART=.TRUE./g' $FDSSTEM*.fds + +# remove the stop file, otherwise the following chain parts +# would not run +rm $CHID.stop \ No newline at end of file diff --git a/propti/lib/__init__.py b/propti/lib/__init__.py index 66f59d7..e2d2e75 100644 --- a/propti/lib/__init__.py +++ b/propti/lib/__init__.py @@ -41,7 +41,7 @@ from .spotpy_wrapper import run_optimisation, create_input_file from .data_structures import Parameter, ParameterSet, \ SimulationSetupSet, SimulationSetup, Relation, DataSource, \ - OptimiserProperties, Version, Sampler + OptimiserProperties, Version, Sampler, Job from .basic_functions import run_simulations, get_save_path from .propti_post_processing import run_best_para diff --git a/propti/lib/data_structures.py b/propti/lib/data_structures.py index 6181b0b..9e108d6 100644 --- a/propti/lib/data_structures.py +++ b/propti/lib/data_structures.py @@ -6,14 +6,14 @@ import numpy as np import pandas as pd import subprocess -from typing import Union import spotpy from .fitness_methods import FitnessMethodInterface -from typing import Union from .. import lib as pr +from typing import Union from typing import List +from typing import Tuple # Reads the script's location. Used to access the propti version number from the # git repo. @@ -1020,17 +1020,13 @@ def __init__(self, db_precision=np.float64): """ Constructor. - :param algorithm: choose sampling algorithm, default: LHS, - range: [LHS,LINEAR] + :param algorithm: choose sampling algorithm, default: LHS, range: [LHS,LINEAR] :param nsamples: number of samples, default: 12 - :param deterministic: If possible, use a deterministic sampling, - default: false - :param seed: If possible, set the seed for the random number generator, - default: None + :param deterministic: If possible, use a deterministic sampling, default: false + :param seed: If possible, set the seed for the random number generator, default: None :param db_name: name of spotpy database file, default: propti_db :param db_type: type of database, default: csv, range: [csv] - :param db_precision: desired precision of the values to be written into - the data base, default: np.float64 + :param db_precision: desired precision of the values to be written into the data base, default: np.float64 """ self.algorithm = algorithm self.nsamples = nsamples @@ -1115,6 +1111,117 @@ def create_sample_set(self, params: ParameterSet) -> List[ParameterSet]: logging.critical("No maching sampler algorithm found.") +############## +# JOB CLASS +class Job: + """ + Class for storing all relevant job parameter, including + - scheduler: wich scheduler should be used e.g. slurm + - template: the path to the template + - parameters: wich placeholder should be changed in the template + """ + + def __init__(self, scheduler: str = "slurm", template: str = "#fds", parameter = []) -> None: + """ + Constructor. + :param scheduler: choose scheduler, default: slurm, range: [slurm] + :param template: path to a template or a default starting with '#', default: #fds, range: [#fds] + :param parameter: list of parameter to replace, default: [] + + parameter can be constructed in different ways: + - str: the string is replaced by the created value from the sampler + - (str, str): the first string is replaced by the second string + - (str, [str]): the first string is replaced by the string inside the list indexed by the sample index + """ + self.scheduler = scheduler + self.template = template + self.parameter = parameter + pass + + def __str__(self) -> str: + """ + Pretty print of (major) class values + :return: str + """ + + template_text = None + if self.template.startswith("#"): + template_text = f"default '{self.template[1:]}'" + else: + template_text = f"'{self.template}'" + + parameter_text = "" + for p in self.parameter: + if type(p) == str: + parameter_text += f"\t'{p}': auto\n" + else: + (name, value) = p + if type(value) == str: + parameter_text += f"\t'{name}' = {value}\n" + else: + parameter_text += f"\t'{name}' = \n" + for (i, item) in enumerate(value): + parameter_text += f"\t {i}:\t{item}\n" + + + text = "\nsampler properties\n" \ + "--------------------\n" \ + f"scheduler: {self.scheduler}\n"\ + f"template: {template_text}\n" \ + f"parameters: \n{parameter_text}"\ + + return text + + def create_jobs(self, execution_dirs: List[str], parameter_sets: List[ParameterSet]): + """ + create a job for each sampled simulation inside its execution directory. + :param execution_dirs: the path to the execution directory. + :param parameter_set: parameters for every simulation. + """ + template = None + template_path = self.template + if os.path.exists(template_path): + file = open(self.template, "r") + template = file.read() + file.close() + else: + sys.exit(f"Template file at '{template_path}' does not exist.\n Use 'propti job template' to clone a predefined template inside the current directory.") + + if len(execution_dirs) != len(parameter_sets): + raise RuntimeError(f"'execution_dirs' ({len(execution_dirs)}) and 'parameter_sets' ({len(parameter_sets)}) have different length.") + + + for i in range(len(execution_dirs)): + sample_template = template + for p in self.parameter: + set_parameter = None + set_value = None + if type(p) == str: + parameter_set = parameter_sets[i] + index = parameter_set.get_index_by_name(p) + set_parameter = p + if index == None: + sys.exit(f"Tried to replace {p} with an auto generated parameter witch does not exist.") + else: + set_value = parameter_sets[i][index].value + else: + (name, value) = p + if type(value) == str: + set_parameter = name + set_value = value + else: + set_parameter = name + set_value = value[i] + # TODO write in documentation that the parameter in the job file must be padded in # and the parameter in the input file mut not have a # padding + sample_template = sample_template.replace(f"#{set_parameter}#", str(set_value)) + sample_template = sample_template.replace("#ID#", str(i)) + sample_template = sample_template.replace("#EXECUTION_DIRS#", execution_dirs[i]) + path = os.path.join(execution_dirs[i], "job.sh") + print("created job at", os.path.join(execution_dirs[i], "job.sh")) + file = open(path, mode="w+") + file.write(sample_template) + file.close() + ###### # MAIN diff --git a/propti/run/propti_job.py b/propti/run/propti_job.py new file mode 100644 index 0000000..2a4c250 --- /dev/null +++ b/propti/run/propti_job.py @@ -0,0 +1,207 @@ +import argparse +import pickle +import subprocess +import os +import sys + +parser = argparse.ArgumentParser(add_help="Create, start and monitor sampler jobs.") +subparser = parser.add_subparsers() + +create_parser = subparser.add_parser("create", help="create the job files inside the sampler folders") +create_parser.add_argument("--scheduler", help="select wich scheduler is used", choices=["slurm"], type=str) +create_parser.add_argument("--template", help="define a path to a template", type=str) +create_parser.add_argument("--parameters", help="define parameters wich get replaced in the template separate every parameter by ';' \na parameter should be described with 'name' to get a value from the sample \nor 'name,value' to set a constant value.", type=str) +create_parser.set_defaults(mode="create") + +start_parser = subparser.add_parser("start", help="start every sampler job.") +start_parser.set_defaults(mode="start") + +cancel_parser = subparser.add_parser("cancel", help="cancel every running job for this sampler.") +cancel_parser.set_defaults(mode="cancel") + +info_parser = subparser.add_parser("info", help="display information about teh status of every job for this sampler.") +info_parser.set_defaults(mode="info") + +template_parser = subparser.add_parser("template", help="clone a template.") +template_parser.add_argument("--name", help="name of the template", type=str, choices=["fds"]) +template_parser.set_defaults(mode="template") + +cmdl_args = parser.parse_args() + + + +def load_from_picke_file(): + # Check if `propti.pickle.sampler` exists + if not os.path.isfile('propti.pickle.sampler'): + sys.exit("'propti.pickle.sampler' not detected. Script execution stopped.") + + in_file = open("propti.pickle.sampler", 'rb') + (sampler_data, job) = pickle.load(in_file) + return sampler_data, job + + +# Reading a complete file when only the last line is important is inefficient, +# therefore a helper function is created that reads a file in reverse. +# source: https://stackoverflow.com/questions/2301789/how-to-read-a-file-in-reverse-order +def reverse_readline(filename, buf_size=8192): + """ + A generator that returns the lines of a file in reverse order + """ + with open(filename, 'rb') as fh: + segment = None + offset = 0 + fh.seek(0, os.SEEK_END) + file_size = remaining_size = fh.tell() + while remaining_size > 0: + offset = min(file_size, offset + buf_size) + fh.seek(file_size - offset) + buffer = fh.read(min(remaining_size, buf_size)).decode(encoding='utf-8') + remaining_size -= buf_size + lines = buffer.split('\n') + # The first line of the buffer is probably not a complete line so + # we'll save it and append it to the last line of the next buffer + # we read + if segment is not None: + # If the previous chunk starts right from the beginning of line + # do not concat the segment to the last line of new chunk. + # Instead, yield the segment first + if buffer[-1] != '\n': + lines[-1] += segment + else: + yield segment + segment = lines[0] + for index in range(len(lines) - 1, 0, -1): + if lines[index]: + yield lines[index] + # Don't yield None if the file was empty + if segment is not None: + yield segment + +def scheduler_not_supported(name): + sys.exit(f"scheduler {name} is not supported") + +if cmdl_args.mode == "create": + (sampler_data, job) = load_from_picke_file() + + from .. import lib as pr + + parameters = [] + for p in cmdl_args.parameter.split(";"): + s = p.split(",") + if len(s) == 1: + parameters.append(s[0]) + elif len(s) == 2: + (name, value) = s + parameters.append((name, value)) + else: + name = s.pop(0) + parameters.append((name, s)) + + job = pr.Job(cmdl_args.scheduler, cmdl_args.template, parameters) + + for (execution_dirs, sample_sets) in sampler_data: + job.create_jobs(execution_dirs, sample_sets) + + out_file = open('propti.pickle.sampler', 'wb') + pickle.dump((sampler_data, job), out_file) + out_file.close() + +elif cmdl_args.mode == "start": + (sampler_data, job) = load_from_picke_file() + if job == None: + sys.exit("Job settings are missing. Use 'create' to create jobs oder define it inside the sampler input file") + scheduler = job.scheduler + if scheduler == "slurm": + for (execution_dirs, _) in sampler_data: + for execution_dir in execution_dirs: + execution_dir = os.path.abspath(execution_dir) + output = subprocess.check_output(["sbatch", "job.sh"], cwd=execution_dir) + else: + scheduler_not_supported(scheduler) + pass +elif cmdl_args.mode == "cancel": + (_, job) = load_from_picke_file() + scheduler = job.scheduler + if scheduler == "slurm": + output = subprocess.check_output(["squeue", "-o", r"\"%o;%i\""]) + root_path = str.encode("\\\"" + os.path.abspath("")) + for line in output.splitlines(): + if line.startswith(root_path): + line = line[2:-3].decode("utf-8") + (_, job_id) = line.split(";") + subprocess.check_output(["scancel", job_id]) + else: + scheduler_not_supported(scheduler) + pass + +elif cmdl_args.mode == "info": + (sampler_data, job) = load_from_picke_file() + scheduler = job.scheduler + if scheduler == "slurm": + def text(t: str, c: int): + """ + Helper function to align test to the left side. + """ + l = len(t) + if l > c: + return t[l - c:] + else: + return " " * (c - l) + t + + paths = [] + for (execution_dirs, _) in sampler_data: + for execution_dir in execution_dirs: + job_file_path = os.path.abspath(os.path.join(execution_dir, "job.sh")) + paths.append(job_file_path) + total = len(paths) + + output = subprocess.check_output(["squeue", "-o", r"\"%o;%i;%j;%t;%M;%l\""]) + + + root_path = str.encode("\\\"" + os.path.abspath("")) + job_id, job_name, status, time, time_limit = "JOBID", "NAME", "ST", "TIME", "TIME_LIMIT" + print(f"{text(job_name,30)} | {text(job_id,8)} | {text(status,2)} | {text(time,10)} | {text(time_limit,10)}") + for line in output.splitlines(): + if line.startswith(root_path): + line = line[2:-2].decode("utf-8") + (job_path, job_id, _, status, time, time_limit) = line.split(";") + for i in range(len(paths)): + if paths[i] == job_path: + paths.pop(i) + break + job_name = os.path.basename(os.path.dirname(job_path)) + print(f"{text(job_name,30)} | {text(job_id,8)} | {text(status,2)} | {text(time,10)} | {text(time_limit,10)}") + + error = 0 + for path in paths: + job_name = os.path.basename(os.path.dirname(path)) + info_path = None + for name in os.listdir(os.path.dirname(path)): + if name.startswith("stderr"): + info_path = os.path.join(os.path.dirname(path),name) + if info_path != None: + reader = reverse_readline(info_path) + line = next(reader, "") + next(reader, "") + if "FDS completed successfully" in line: + print(f"{text(job_name,30)} | -------- | F | ---------- | ----------") + continue + error += 1 + print(f"{text(job_name,30)} | -------- | E | ---------- | ----------") + if error == 1: + print(f"{len(paths)}/{total} finished with {error} error.") + else: + print(f"{len(paths)}/{total} finished with {error} errors.") + else: + scheduler_not_supported(scheduler) + pass + +elif cmdl_args.mode == "template": + path = os.path.join("/".join(__loader__.path.split("/")[:-2]), "jobs", cmdl_args.name) + if os.path.exists(path): + with open(path) as read: + with open("job_template","w") as write: + write.write(read.read()) + print("Copied template as 'job_template'.") + else: + sys.exit(f"A template with the name '{cmdl_args.name}' does not exist") + diff --git a/propti/run/propti_sampling.py b/propti/run/propti_sampling.py index d77da2e..3893b63 100644 --- a/propti/run/propti_sampling.py +++ b/propti/run/propti_sampling.py @@ -1,11 +1,9 @@ import sys import os import typing -import numpy as np import copy -import pandas as pd import shutil as sh -import scipy +import pickle from .. import lib as pr @@ -25,6 +23,7 @@ setups: pr.SimulationSetupSet = None params: pr.ParameterSet = None sampler: pr.Sampler = None +job: pr.Job = None input_file = cmdl_args.input_file @@ -80,7 +79,13 @@ logging.info(setups) logging.info(params) logging.info(sampler) +if job != None: + logging.info(job) +else: + logging.info("no job settings in input file") +# Build simulation and jobs +pickle_data = [] for simulation in setups: os.makedirs(simulation.execution_dir_prefix,exist_ok=True) sample_set = sampler.create_sample_set(simulation.model_parameter) @@ -109,8 +114,9 @@ para_table_file.write(line_max + '\n') logging.debug("Sample set:\n") - sample_index = 0 - for sample in sample_set: + + execution_dirs = [] + for (sample_index, sample) in enumerate(sample_set): logging.debug(f" {sample}") line = f"\t\t{sample_index:06d}" @@ -124,13 +130,21 @@ tmp_simulation_setup = typing.cast(pr.SimulationSetup, copy.deepcopy(simulation)) tmp_simulation_setup.model_parameter = sample tmp_simulation_setup.execution_dir = sample.name + execution_dirs.append(os.path.join(tmp_simulation_setup.execution_dir_prefix, tmp_simulation_setup.execution_dir)) os.makedirs(os.path.join(tmp_simulation_setup.execution_dir_prefix, tmp_simulation_setup.execution_dir), exist_ok=True) pr.create_input_file(tmp_simulation_setup) - sample_index += 1 + pickle_data.append((execution_dirs, sample_set)) + # Create job files + if job != None: + job.create_jobs(execution_dirs, sample_set) para_table_file.close() +out_file = open('propti.pickle.sampler', 'wb') +pickle.dump((pickle_data, job), out_file) +out_file.close() + # if cmdl_args.prepare_init_inputs: # logging.info("prepare input files with initial values") # for s in setups: diff --git a/propti/run/propti_sjob.py b/propti/run/propti_sjob.py deleted file mode 100644 index 3bfef00..0000000 --- a/propti/run/propti_sjob.py +++ /dev/null @@ -1,264 +0,0 @@ -import argparse -import subprocess -import os -import re - -parser = argparse.ArgumentParser(add_help="Create, start and monitor sampler jobs for Slurm.") -subparser = parser.add_subparsers() -create_parser = subparser.add_parser("create", help="create the job files inside the sampler folders") -create_parser.add_argument("--time", default="0-02:00:00", help="the maximal time the job can run in the following format: days-hours:minutes:seconds"); -create_parser.set_defaults(mode="create") -start_parser = subparser.add_parser("start", help="start every sampler job (note: the create command had to be run)") -start_parser.set_defaults(mode="start") -start_parser = subparser.add_parser("cancel", help="cancel every running job for this sampler") -start_parser.set_defaults(mode="cancel") -start_parser = subparser.add_parser("info", help="start jobs") -start_parser.set_defaults(mode="info") -parser.add_argument("root_dir", type=str, - help="the folder wich contains all sampler folders (note: the folder name usually named like 'sample_[...]' )", default='.') -cmdl_args = parser.parse_args() - -root_dir = cmdl_args.root_dir - -def get_table_data(): - """ - Opens the CSV table created by the sampler, reads its contents, and extracts relevant data - such as file_name, name, and a list of ids. - - Returns: - tuple: A tuple containing three elements - file_name (str), name (str), and ids (list[str]). - - file_name: The name of the original file (with extension). - - name: The CHID name for the simulations. - - ids: The ids of every simulation - """ - sample_table_path = os.path.join(cmdl_args.root_dir, "sample_table.csv") - sample_table_file = open(sample_table_path) - - file_name = sample_table_file.readline()[13:-1] - name = file_name.replace(".fds","") - search = re.search("chid\s*=\s*(.*)",sample_table_file.readline()) - if search: - name = search.group(1) - ids = [] - for line in sample_table_file: - if line.startswith("#") or line == "": - continue - else: - ids.append(line.split(",")[0].strip()) - sample_table_file.close() - return file_name,name, ids - - -# Reading a complete file when only the last line is important is inefficient, -# therefore a helper function is created that reads a file in reverse. -# source: https://stackoverflow.com/questions/2301789/how-to-read-a-file-in-reverse-order -def reverse_readline(filename, buf_size=8192): - """ - A generator that returns the lines of a file in reverse order - """ - with open(filename, 'rb') as fh: - segment = None - offset = 0 - fh.seek(0, os.SEEK_END) - file_size = remaining_size = fh.tell() - while remaining_size > 0: - offset = min(file_size, offset + buf_size) - fh.seek(file_size - offset) - buffer = fh.read(min(remaining_size, buf_size)).decode(encoding='utf-8') - remaining_size -= buf_size - lines = buffer.split('\n') - # The first line of the buffer is probably not a complete line so - # we'll save it and append it to the last line of the next buffer - # we read - if segment is not None: - # If the previous chunk starts right from the beginning of line - # do not concat the segment to the last line of new chunk. - # Instead, yield the segment first - if buffer[-1] != '\n': - lines[-1] += segment - else: - yield segment - segment = lines[0] - for index in range(len(lines) - 1, 0, -1): - if lines[index]: - yield lines[index] - # Don't yield None if the file was empty - if segment is not None: - yield segment - - - -if cmdl_args.mode == "create": - # The The execution script for the simulation. Variables padded with # get replaced by the corresponding value, for the specific simulation. - job_file_content = """#!/bin/bash -#!/bin/sh -# Name of the Job -#SBATCH --job-name=#NAME# - -# On witch device the simulation is run -#SBATCH --partition=normal - -# Maximum time the job can run: days-hours:minutes:seconds -#SBATCH --time=#TIME# - -# Number of cores -#SBATCH --tasks-per-node=1 -#SBATCH --cpus-per-task=1 -#SBATCH --nodes=#NODES# - -# Output file name -#SBATCH --output=stdout.%j -#SBATCH --error=stderr.%j - - -cd #PATH# - -mkdir ./results -cd ./results - -# define FDS input file -FDSSTEM=../ - -# grep the CHID (used for stop file) -CHID=`sed -n "s/^.*CHID='\\([-0-9a-zA-Z_]*\\)'.*$/\\1/p" < $FDSSTEM*.fds` - -# append the start time to file 'time_start' -echo "$SLURM_JOB_ID -- `date`" >> time_start - -# handle the signal sent before the end of the wall clock time -function handler_sigusr1 -{ - # protocol stopping time - echo "$SLURM_JOB_ID -- `date`" >> time_stop - echo "`date` Shell received stop signal" - - # create FDS stop file - touch $CHID.stop - - # as manual stop was triggered, the end of simulation time was - # not reached, remove flag file 'simulation_time_end' - rm simulation_time_end - wait -} - -# register the function 'hander_sigusr1' to handle the signal send out -# just before the end of the wall clock time -trap "handler_sigusr1" SIGUSR1 - -# check for the simulation finished flag file 'simulation_time_end' -# if it is found, just quit -if [ -e simulation_time_end ]; then - ## simulation has already finished, nothing left to do - echo "FDS simulation already finished" - exit 0 -fi - -# simulation not finished yet -# create flag file to check for reaching simulation end time -touch simulation_time_end - -# Load FDS -module use -a /beegfs/larnold/modules -module load FDS - -# set the number of OMP threads -export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} - -# run FDS executable -mpiexec fds $FDSSTEM*.fds & wait - -# TODO make this more robust -# set RESTART to TRUE in FDS input file -sed -i 's/RESTART\s*=\s*(F|.FALSE.)/RESTART=.TRUE./g' $FDSSTEM*.fds - -# remove the stop file, otherwise the following chain parts -# would not run -rm $CHID.stop - """ - - - fds_file_name, name, ids = get_table_data() - for id in ids: - directory =os.path.join(root_dir,f"sample_{id}") - fds_file_path = os.path.join(directory, fds_file_name) - job_file_path = os.path.join(directory, "job.sh") - processes = 1 - fds_file = open(fds_file_path) - for line in fds_file: - line = line.strip() - if line.startswith("&MESH"): - search = re.search("MPI_PROCESS\s*=\s*([0-9]+)", line) - if search: - processes = int(search.group(1)) + 1 - process_name = f"{name}_{id}" - fds_file.close() - job_file = open(job_file_path, "w") - job = job_file_content.replace("#NAME#", process_name).replace("#TIME#", cmdl_args.time).replace("#NODES#", str(processes)).replace("#PATH#", f"\"{directory}\"") - job_file.write(job) - job_file.close() - -elif cmdl_args.mode == "start": - fds_file_name, name, ids = get_table_data() - for id in ids: - job_file_path = os.path.abspath(os.path.join(root_dir,f"sample_{id}", "job.sh")) - output = subprocess.check_output(["sbatch", job_file_path]) -elif cmdl_args.mode == "cancel": - output = subprocess.check_output(["squeue", "-o", r"\"%o;%i\""]) - root_path = str.encode("\\\"" + os.path.abspath("")) - print(root_path) - for line in output.splitlines(): - if line.startswith(root_path): - line = line[2:-3].decode("utf-8") - (_, id) = line.split(";") - subprocess.check_output(["scancel", id]) - -elif cmdl_args.mode == "info": - def text(t, c): - l = len(t) - if l > c: - return t[l - c:] - else: - return " " * (c - l) + t - - paths = [] - fds_file_name, name, ids = get_table_data() - for id in ids: - job_file_path = os.path.abspath(os.path.join(root_dir,f"sample_{id}", "job.sh")) - paths.append((job_file_path, id)) - total = len(paths) - - output = subprocess.check_output(["squeue", "-o", r"\"%o;%i;%j;%t;%M;%l\""]) - - root_path = str.encode("\\\"" + os.path.abspath("")) - print(root_path) - id, job_name, status, time, time_limit = "JOBID","NAME", "ST", "TIME", "TIME_LIMIT" - print(f"{text(job_name,30)} | {text(id,8)} | {text(status,2)} | {text(time,10)} | {text(time_limit,10)}") - for line in output.splitlines(): - if line.startswith(root_path): - line = line[2:-3].decode("utf-8") - (job_path, id, job_name, status, time, time_limit) = line.split(";") - print(f"{text(job_name,30)} | {text(id,8)} | {text(status,2)} | {text(time,10)} | {text(time_limit,10)}") - for i in range(len(paths)): - if paths[i][0] == job_path: - paths.pop(i) - break - - error = 0 - for (path, id) in paths: - info_path = os.path.join(os.path.dirname(path),"results",f"{name}.out") - if os.path.exists(info_path): - reader = reverse_readline(info_path) - line = next(reader, "") - if "FDS completed successfully" in line: - job_name, status = name + "_" + id, "F" - else: - job_name, status = name + "_" + id, "E " # Indent for easier spotting - error += 1 - else: - job_name, status = name + "_" + id, "E " # Indent for easier spotting - error += 1 - print(f"{text(job_name,30)} | -------- | {text(status,2)} | ---------- | ----------") - if error == 1: - print(f"{len(paths)}/{total} finished with {error} error.") - else: - print(f"{len(paths)}/{total} finished with {error} errors.") \ No newline at end of file diff --git a/setup.py b/setup.py index 8e6ab05..95588c1 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,18 @@ with open(os.path.join(base_dir, "propti", "__init__.py"), 'r', encoding="utf-8") as f: version = f.readline().split("=")[-1].strip()[1:-1] +data_files = [] +def get_data_files(path, data_files): + for p in os.listdir(path): + p = os.path.join(path, p) + if os.path.isdir(p): + get_data_files(p, data_files) + else: + data_files.append(p) +get_data_files("propti/jobs", data_files) +print(data_files) + + setuptools.setup( name="propti", version=version, @@ -23,8 +35,9 @@ "Operating System :: OS Independent", ], python_requires='>=3.0', - #package_dir={"propti": "propti/"}, - packages = ["propti", "propti/lib", "propti/run"], + include_package_data=True, + data_files=data_files, + packages = ["propti", "propti/lib", "propti/run", "propti/jobs"], install_requires= [ "numpy", From 95b424581ffcade92fc62e0e0a85e4fbdb1aef9f Mon Sep 17 00:00:00 2001 From: Xanthron Writer Date: Mon, 28 Aug 2023 21:24:23 +0200 Subject: [PATCH 23/23] Updated the example and added an instruction --- examples/sampling_lhs_01/cone_template.fds | 4 +- examples/sampling_lhs_01/input.py | 13 ++++- examples/sampling_lhs_01/instruction.md | 66 ++++++++++++++++++++++ 3 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 examples/sampling_lhs_01/instruction.md diff --git a/examples/sampling_lhs_01/cone_template.fds b/examples/sampling_lhs_01/cone_template.fds index ad1997a..bca3b2f 100644 --- a/examples/sampling_lhs_01/cone_template.fds +++ b/examples/sampling_lhs_01/cone_template.fds @@ -1,10 +1,10 @@ &HEAD CHID='#CHID#', TITLE='Example from FDS user guide' / -&MESH IJK=3,3,3, XB=-0.15,0.15,-0.15,0.15,0.0,0.3 / +&MESH IJK=3,3,3, XB=-0.15,0.15,-0.15,0.15,0.0,0.3, MPI_PROCESS = 0 / &TIME T_END=600.0, WALL_INCREMENT=1, DT=0.05 / -&MISC SOLID_PHASE_ONLY=.TRUE. / +&MISC SOLID_PHASE_ONLY=.TRUE., RESTART=.TRUE./ &SPEC ID='METHANE' / &MATL ID='BLACKPMMA' diff --git a/examples/sampling_lhs_01/input.py b/examples/sampling_lhs_01/input.py index e59abe3..4060298 100644 --- a/examples/sampling_lhs_01/input.py +++ b/examples/sampling_lhs_01/input.py @@ -37,6 +37,13 @@ setups.append(s) -# use default values for optimiser -sampler = pr.Sampler(algorithm='LHS', - nsamples=10) \ No newline at end of file +nsamples = 5 +sampler = pr.Sampler(algorithm='LINEAR', + nsamples=nsamples) +time = [] +for i in range(nsamples): + time.append(f"0-00:1{i}:00") +job = pr.Job(template="fds", parameter=[ + ("CHID",CHID), + ("TIME",time), + ("NODES","1")]) \ No newline at end of file diff --git a/examples/sampling_lhs_01/instruction.md b/examples/sampling_lhs_01/instruction.md new file mode 100644 index 0000000..d1c51cd --- /dev/null +++ b/examples/sampling_lhs_01/instruction.md @@ -0,0 +1,66 @@ +# Instruction +An example for using the sampler to create multiple simulations from one input file with different parameters. +## 1. Clone a Template File +```bash +propti template --name fds +``` + +By utilizing the `name` variable, it is possible to clone a template from the directory `propti/jobs` into the ongoing working directory. This permits flexible customization of simulations through parameter replacement, denoted by # placeholders. + +Template Example: +```bash +#!/bin/bash +#!/bin/sh +# Name of the Job +#SBATCH --job-name=#CHID#_#ID# + +# On witch device the simulation is run +#SBATCH --partition=norma +``` + +The above exemplifies a template script sourced from `propti/jobs/fds`. In this script, `#CHID#` and `#ID#` serve as variables that undergo replacement. While `#CHID#` is a variable requiring definition within the input file, `#ID#` is automatically substituted. + + +## 2. Create an `input.py` File + +A python file that utilizes the propti module is necessary. For an idea how to structure the input file see `input.py`. It is important that the sampler is created before the jobs. + +The sampler has currently 2 significant setup parameters: +- `algorithm`: Specifies the algorithm utilized for parameter computation. Currently supported: `LHS` and `LINEAR` +- `nsamples`: Defines the amount of samples that are generated. + +To automatically crate jobs there are 3 important settings: +- `scheduler`: set the scheduler that is used. currently only `slurm` is supported, serving as the default value. +- `template`: Refers to the path leading to the template file. +- `parameters`: Defines parameters slated for replacement. There are 3 possible ways to define a parameter: + - `NAME` Solely the parameter name is provided. Subsequent substitution involves the parameter generated by the sampler for the ongoing simulation. + - (`NAME`, `VALUE`) Each placeholder is replaced by the same value + - (`NAME`, `LIST[VALUE]`) Each placeholder is replaced by the corresponding value. substituted by corresponding values. It is essential for the list's length to match 'nsamples'. + + + +## 3. Initiate the Sampler +Execute the subsequent command to launch the input file, thereby initiating simulation creation alongside job execution scripts. +```bash +propti sampler input.py +``` +Once simulations are ready for execution, activate the subsequent command: +```bash +propti job start +``` +## 4. Query Running Job Information +To acquire real-time insight into job statuses, execute the following command: +```bash +propti job info +``` + +The output could look something like this. +``` + NAME | JOBID | ST | TIME | TIME_LIMIT +sample_000000 | 19932566 | R | 7:17 | 20:00 +sample_000003 | 19932569 | R | 7:17 | 20:00 +sample_000001 | -------- | F | ---------- | ---------- +sample_000002 | -------- | F | ---------- | ---------- +sample_000004 | -------- | F | ---------- | ---------- +3/5 finished with 0 errors. +``` \ No newline at end of file