Replace parallel easyconfig parameter by maxparallel

easybuild/framework/easyblock.py

@@ -1890,7 +1890,7 @@ def skip_extensions_parallel(self, exts_filter):
         exts_cnt = len(self.ext_instances)
         cmds = [resolve_exts_filter_template(exts_filter, ext) for ext in self.ext_instances]
 
-        with ThreadPoolExecutor(max_workers=self.cfg['parallel']) as thread_pool:
+        with ThreadPoolExecutor(max_workers=self.cfg.parallel) as thread_pool:
 
             # list of command to run asynchronously
             async_cmds = [thread_pool.submit(run_shell_cmd, cmd, stdin=stdin, hidden=True, fail_on_error=False,
@@ -2020,7 +2020,7 @@ def install_extensions_parallel(self, install=True):
         """
         self.log.info("Installing extensions in parallel...")
 
-        thread_pool = ThreadPoolExecutor(max_workers=self.cfg['parallel'])
+        thread_pool = ThreadPoolExecutor(max_workers=self.cfg.parallel)
 
         running_exts = []
         installed_ext_names = []
@@ -2082,7 +2082,7 @@ def update_exts_progress_bar_helper(running_exts, progress_size):
 
             for _ in range(max_iter):
 
-                if not (exts_queue and len(running_exts) < self.cfg['parallel']):
+                if not (exts_queue and len(running_exts) < self.cfg.parallel):
                     break
 
                 # check whether extension at top of the queue is ready to install
@@ -2321,19 +2321,12 @@ def set_parallel(self):
         """Set 'parallel' easyconfig parameter to determine how many cores can/should be used for parallel builds."""
         # set level of parallelism for build
         par = build_option('parallel')
-        cfg_par = self.cfg['parallel']
-        if cfg_par is None:
+        if par is not None:
             self.log.debug("Desired parallelism specified via 'parallel' build option: %s", par)
-        elif par is None:
-            par = cfg_par
-            self.log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par)
-        else:
-            par = min(int(par), int(cfg_par))
-            self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par)
 
         par = det_parallelism(par, maxpar=self.cfg['maxparallel'])
         self.log.info("Setting parallelism: %s" % par)
-        self.cfg['parallel'] = par
+        self.cfg.parallel = par
 
     def remove_module_file(self):
         """Remove module file (if it exists), and check for ghost installation directory (and deal with it)."""

easybuild/framework/easyconfig/default.py

@@ -110,8 +110,6 @@
     'installopts': ['', 'Extra options for installation', BUILD],
     'maxparallel': [16, 'Max degree of parallelism', BUILD],
     'module_only': [False, 'Only generate module file', BUILD],
-    'parallel': [None, ('Degree of parallelism for e.g. make (default: based on the number of '
-                        'cores, active cpuset and restrictions in ulimit)'), BUILD],
     'patches': [[], "List of patches to apply", BUILD],
     'prebuildopts': ['', 'Extra options pre-passed to build command.', BUILD],
     'preconfigopts': ['', 'Extra options pre-passed to configure.', BUILD],

easybuild/framework/easyconfig/easyconfig.py

@@ -127,7 +127,8 @@ def new_ec_method(self, key, *args, **kwargs):
             key, ver = DEPRECATED_EASYCONFIG_PARAMETERS[depr_key]
             _log.deprecated("Easyconfig parameter '%s' is deprecated, use '%s' instead" % (depr_key, key), ver)
         elif key in REPLACED_PARAMETERS:
-            _log.nosupport("Easyconfig parameter '%s' is replaced by '%s'" % (key, REPLACED_PARAMETERS[key]), '2.0')
+            newkey, ver = REPLACED_PARAMETERS[key]
+            _log.nosupport("Easyconfig parameter '%s' is replaced by '%s'" % (key, newkey), ver)
         return ec_method(self, key, *args, **kwargs)
 
     return new_ec_method
@@ -499,6 +500,9 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi
         self.iterate_options = []
         self.iterating = False
 
+        # Storage for parallel property. Mark as unset initially
+        self._parallel = None
+
         # parse easyconfig file
         self.build_specs = build_specs
         self.parse()
@@ -1212,6 +1216,20 @@ def all_dependencies(self):
 
         return self._all_dependencies
 
+    @property
+    def parallel(self):
+        """Number of parallel jobs to be used for building etc."""
+        if self._parallel is None:
+            raise EasyBuildError("Parallelism in EasyConfig not set yet. "
+                                 "Need to call the easyblocks set_parallel first.")
+        return self._parallel
+
+    @parallel.setter
+    def parallel(self, value):
+        # Update backstorage and template value
+        self._parallel = value
+        self.template_values['parallel'] = value
+
     def dump(self, fp, always_overwrite=True, backup=False, explicit_toolchains=False):
         """
         Dump this easyconfig to file, with the given filename.

easybuild/framework/easyconfig/format/format.py

@@ -68,7 +68,7 @@
     ['preconfigopts', 'configopts'],
     ['prebuildopts', 'buildopts'],
     ['preinstallopts', 'installopts'],
-    ['parallel', 'maxparallel'],
+    ['maxparallel'],
 ]
 LAST_PARAMS = ['exts_default_options', 'exts_list',
                'sanity_check_paths', 'sanity_check_commands',

easybuild/framework/easyconfig/parser.py

@@ -107,9 +107,11 @@
 
 # replaced easyconfig parameters, and their replacements
 REPLACED_PARAMETERS = {
-    'license': 'license_file',
-    'makeopts': 'buildopts',
-    'premakeopts': 'prebuildopts',
+    # <old_param>: (<new_param>, <removed_in_version>),
+    'license': ('license_file', '2.0'),
+    'makeopts': ('buildopts', '2.0'),
+    'parallel': ('maxparallel', '5.0'),
+    'premakeopts': ('prebuildopts', '2.0'),
 }
 
 _log = fancylogger.getLogger('easyconfig.parser', fname=False)

easybuild/framework/easyconfig/templates.py

@@ -59,7 +59,6 @@
     'bitbucket_account',
     'github_account',
     'name',
-    'parallel',
     'version',
     'versionsuffix',
     'versionprefix',
@@ -108,7 +107,7 @@
     'software_commit': "Git commit id to use for the software as specified by --software-commit command line option",
     'sysroot': "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include"
                "as specify by the --sysroot configuration option",
-
+    'parallel': "Degree of parallelism for e.g. make",
 }
 
 # constant templates that can be used in easyconfigs

easybuild/scripts/mk_tmpl_easyblock_for.py

@@ -165,7 +165,7 @@ def build_step(self):
         comp_fam = comp_map[self.toolchain.comp_family()]
 
         # enable parallel build
-        par = self.cfg['parallel']
+        par = self.cfg.parallel
         cmd = "build command --parallel %%d --compiler-family %%s" %% (par, comp_fam)
         run_shell_cmd(cmd)
 

easybuild/tools/systemtools.py

@@ -1214,6 +1214,8 @@ def get_default_parallelism():
             raise EasyBuildError("Specified level of parallelism '%s' is not an integer value: %s", par, err)
 
     if maxpar is not None and maxpar < par:
+        if maxpar is False:
+            maxpar = 1
         _log.info("Limiting parallelism from %s to %s", par, maxpar)
         par = maxpar
 

test/framework/easyblock.py

@@ -2219,58 +2219,83 @@ def test_parallel(self):
         toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
         toytxt = read_file(toy_ec)
 
+        handle, toy_ec_error = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb')
+        os.close(handle)
+        write_file(toy_ec_error, toytxt + "\nparallel = 123")
+
         handle, toy_ec1 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb')
         os.close(handle)
-        write_file(toy_ec1, toytxt + "\nparallel = 13")
+        write_file(toy_ec1, toytxt + "\nmaxparallel = None")
 
         handle, toy_ec2 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb')
         os.close(handle)
-        write_file(toy_ec2, toytxt + "\nparallel = 123\nmaxparallel = 67")
+        write_file(toy_ec2, toytxt + "\nmaxparallel = 6")
 
         handle, toy_ec3 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb')
         os.close(handle)
-        write_file(toy_ec3, toytxt + "\nparallel = False")
+        write_file(toy_ec3, toytxt + "\nmaxparallel = False")
+
+        import easybuild.tools.systemtools as st
+        auto_parallel = 15
+        st.det_parallelism._default_parallelism = auto_parallel
+
+        # 'parallel' easyconfig parameter specified is an error
+        self.assertRaises(EasyBuildError, EasyConfig, toy_ec_error)
+        self.assertErrorRegex(EasyBuildError, "Easyconfig parameter 'parallel' is replaced by 'maxparallel'",
+                              EasyConfig, toy_ec_error)
 
         # default: parallelism is derived from # available cores + ulimit
+        # Note that maxparallel has a default of 16, so we need a lower auto_paralle value here
         test_eb = EasyBlock(EasyConfig(toy_ec))
         test_eb.check_readiness_step()
-        self.assertTrue(isinstance(test_eb.cfg['parallel'], int) and test_eb.cfg['parallel'] > 0)
+        self.assertEqual(test_eb.cfg.parallel, auto_parallel)
 
-        # only 'parallel' easyconfig parameter specified (no 'parallel' build option)
-        test_eb = EasyBlock(EasyConfig(toy_ec1))
-        test_eb.check_readiness_step()
-        self.assertEqual(test_eb.cfg['parallel'], 13)
+        auto_parallel = 128  # Don't limit by available CPU cores mock
+        st.det_parallelism._default_parallelism = auto_parallel
 
-        # both 'parallel' and 'maxparallel' easyconfig parameters specified (no 'parallel' build option)
+        # only 'maxparallel' easyconfig parameter specified (no 'parallel' build option)
         test_eb = EasyBlock(EasyConfig(toy_ec2))
         test_eb.check_readiness_step()
-        self.assertEqual(test_eb.cfg['parallel'], 67)
+        self.assertEqual(test_eb.cfg.parallel, 6)
 
-        # make sure 'parallel = False' is not overriden (no 'parallel' build option)
+        # make sure 'maxparallel = False' is treated as 1 (no 'parallel' build option)
         test_eb = EasyBlock(EasyConfig(toy_ec3))
         test_eb.check_readiness_step()
-        self.assertEqual(test_eb.cfg['parallel'], False)
+        self.assertEqual(test_eb.cfg.parallel, 1)
 
         # only 'parallel' build option specified
-        init_config(build_options={'parallel': '9', 'validate': False})
+        init_config(build_options={'parallel': '13', 'validate': False})
         test_eb = EasyBlock(EasyConfig(toy_ec))
         test_eb.check_readiness_step()
-        self.assertEqual(test_eb.cfg['parallel'], 9)
+        self.assertEqual(test_eb.cfg.parallel, 13)
 
-        # both 'parallel' build option and easyconfig parameter specified (no 'maxparallel')
+        # 'maxparallel = None' is the same as unset
         test_eb = EasyBlock(EasyConfig(toy_ec1))
         test_eb.check_readiness_step()
-        self.assertEqual(test_eb.cfg['parallel'], 9)
+        self.assertEqual(test_eb.cfg.parallel, 13)
 
-        # both 'parallel' and 'maxparallel' easyconfig parameters specified + 'parallel' build option
+        # 'maxparallel' easyconfig parameter with 'parallel' build option
         test_eb = EasyBlock(EasyConfig(toy_ec2))
         test_eb.check_readiness_step()
-        self.assertEqual(test_eb.cfg['parallel'], 9)
+        self.assertEqual(test_eb.cfg.parallel, 6)
 
-        # make sure 'parallel = False' is not overriden (with 'parallel' build option)
+        # make sure 'maxparallel = False' is treated as 1 (with 'parallel' build option)
         test_eb = EasyBlock(EasyConfig(toy_ec3))
         test_eb.check_readiness_step()
-        self.assertEqual(test_eb.cfg['parallel'], 0)
+        self.assertEqual(test_eb.cfg.parallel, 1)
+
+        # Template updated correctly
+        test_eb.cfg['buildopts'] = '-j %(parallel)s'
+        self.assertEqual(test_eb.cfg['buildopts'], '-j 1')
+        # Might be done in an easyblock step
+        test_eb.cfg.parallel = 42
+        self.assertEqual(test_eb.cfg['buildopts'], '-j 42')
+        # Unaffected by build settings
+        test_eb.cfg.parallel = 421337
+        self.assertEqual(test_eb.cfg['buildopts'], '-j 421337')
+
+        # Reset mocked value
+        del st.det_parallelism._default_parallelism
 
     def test_guess_start_dir(self):
         """Test guessing the start dir."""

test/framework/easyconfig.py

@@ -671,7 +671,7 @@ def test_tweaking(self):
             'version = "3.14"',
             'toolchain = {"name": "GCC", "version": "4.6.3"}',
             'patches = %s',
-            'parallel = 1',
+            'maxparallel = 1',
             'keepsymlinks = True',
         ]) % str(patches)
         self.prep()
@@ -694,7 +694,7 @@ def test_tweaking(self):
             # It should be possible to overwrite values with True/False/None as they often have special meaning
             'runtest': 'False',
             'hidden': 'True',
-            'parallel': 'None',  # Good example: parallel=None means "Auto detect"
+            'maxparallel': 'None',  # Good example: maxparallel=None means "unlimitted"
             # Adding new options (added only by easyblock) should also be possible
             # and in case the string "True/False/None" is really wanted it is possible to quote it first
             'test_none': '"False"',
@@ -711,7 +711,7 @@ def test_tweaking(self):
         self.assertEqual(eb['patches'], new_patches)
         self.assertIs(eb['runtest'], False)
         self.assertIs(eb['hidden'], True)
-        self.assertIsNone(eb['parallel'])
+        self.assertIsNone(eb['maxparallel'])
         self.assertEqual(eb['test_none'], 'False')
         self.assertEqual(eb['test_bool'], 'True')
         self.assertEqual(eb['test_123'], 'None')
@@ -1931,8 +1931,7 @@ def test_alternative_easyconfig_parameters(self):
 
     def test_deprecated_easyconfig_parameters(self):
         """Test handling of deprecated easyconfig parameters."""
-        os.environ.pop('EASYBUILD_DEPRECATED')
-        easybuild.tools.build_log.CURRENT_VERSION = self.orig_current_version
+        self.allow_deprecated_behaviour()
         init_config()
 
         test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')
@@ -3517,7 +3516,6 @@ def test_template_constant_dict(self):
             'namelower': 'gzip',
             'nameletter': 'g',
             'nameletterlower': 'g',
-            'parallel': None,
             'rpath_enabled': rpath,
             'software_commit': '',
             'sysroot': '',
@@ -3556,7 +3554,6 @@ def test_template_constant_dict(self):
 
         res = template_constant_dict(ec)
         res.pop('arch')
-        expected['parallel'] = 12
         self.assertEqual(res, expected)
 
         toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-deps.eb')
@@ -3598,7 +3595,6 @@ def test_template_constant_dict(self):
             'toolchain_name': 'system',
             'toolchain_version': 'system',
             'nameletterlower': 't',
-            'parallel': None,
             'pymajver': '3',
             'pyminver': '7',
             'pyshortver': '3.7',
@@ -3634,7 +3630,7 @@ def test_template_constant_dict(self):
         ec = EasyConfigParser(filename=test_ec).get_config_dict()
 
         expected['module_name'] = None
-        for key in ('bitbucket_account', 'github_account', 'parallel', 'versionprefix'):
+        for key in ('bitbucket_account', 'github_account', 'versionprefix'):
             del expected[key]
 
         dep_names = [x[0] for x in ec['dependencies']]

test/framework/easyconfigs/test_ecs/s/SQLite/SQLite-3.8.10.2-GCC-6.4.0-2.28.eb

@@ -31,7 +31,7 @@ dependencies = [
     # ('Tcl', '8.6.4'),
 ]
 
-parallel = 1
+maxparallel = 1
 
 sanity_check_paths = {
     'files': ['bin/sqlite3', 'include/sqlite3ext.h', 'include/sqlite3.h', 'lib/libsqlite3.a', 'lib/libsqlite3.so'],

test/framework/easyconfigs/test_ecs/s/SQLite/SQLite-3.8.10.2-foss-2018a.eb

@@ -31,7 +31,7 @@ dependencies = [
     # ('Tcl', '8.6.4'),
 ]
 
-parallel = 1
+maxparallel = 1
 
 sanity_check_paths = {
     'files': ['bin/sqlite3', 'include/sqlite3ext.h', 'include/sqlite3.h', 'lib/libsqlite3.a', 'lib/libsqlite3.so'],

test/framework/easyconfigs/test_ecs/s/SQLite/SQLite-3.8.10.2-gompi-2018a.eb

@@ -31,7 +31,7 @@ dependencies = [
     # ('Tcl', '8.6.4'),
 ]
 
-parallel = 1
+maxparallel = 1
 
 sanity_check_paths = {
     'files': ['bin/sqlite3', 'include/sqlite3ext.h', 'include/sqlite3.h', 'lib/libsqlite3.a', 'lib/libsqlite3.so'],

test/framework/easyconfigs/test_ecs/s/ScaLAPACK/ScaLAPACK-2.0.2-gompi-2018a-OpenBLAS-0.2.20-broken.eb

@@ -23,6 +23,6 @@ builddependencies = [('CMake','2.8.10')]
 dependencies = [(local_blaslib, local_blasver, '')]
 
 # parallel build tends to fail, so disabling it
-parallel = 1
+maxparallel = 1
 
 moduleclass = 'numlib'

test/framework/easyconfigs/test_ecs/s/ScaLAPACK/ScaLAPACK-2.0.2-gompi-2018a-OpenBLAS-0.2.20.eb

@@ -22,6 +22,6 @@ versionsuffix = "-%s-%s" % (local_blaslib, local_blasver)
 dependencies = [(local_blaslib, local_blasver, '')]
 
 # parallel build tends to fail, so disabling it
-parallel = 1
+maxparallel = 1
 
 moduleclass = 'numlib'

test/framework/easyconfigs/test_ecs/s/ScaLAPACK/ScaLAPACK-2.0.2-gompic-2018a-OpenBLAS-0.2.20.eb

@@ -22,6 +22,6 @@ versionsuffix = "-%s-%s" % (local_blaslib, local_blasver)
 dependencies = [(local_blaslib, local_blasver)]
 
 # parallel build tends to fail, so disabling it
-parallel = 1
+maxparallel = 1
 
 moduleclass = 'numlib'

test/framework/utilities.py

@@ -210,6 +210,14 @@ def allow_deprecated_behaviour(self):
         os.environ.pop('EASYBUILD_DEPRECATED', None)
         eb_build_log.CURRENT_VERSION = self.orig_current_version
 
+    @contextmanager
+    def temporarily_allow_deprecated_behaviour(self):
+        self.allow_deprecated_behaviour()
+        try:
+            yield
+        finally:
+            self.disallow_deprecated_behaviour()
+
     @contextmanager
     def log_to_testlogfile(self):
         """Context manager class to capture log output in self.logfile for the scope used. Clears the file first"""