give a try to ruamel.yaml

Author: albestro

requirements.txt

@@ -1,4 +1,4 @@
 Jinja2
 jsonschema
 pytest
-PyYAML
+ruamel.yaml >=0.15.1, <0.18.0

stackinator/builder.py

@@ -9,10 +9,12 @@
 from datetime import datetime
 
 import jinja2
-import yaml
+from ruamel.yaml import YAML
 
 from . import VERSION, cache, root_logger, spack_util
 
+yaml = YAML()
+
 
 def install(src, dst, *, ignore=None, symlinks=False):
     """Call shutil.copytree or shutil.copy2. copy2 is used if `src` is not a directory.
@@ -345,7 +347,7 @@ def generate(self, recipe):
         if repo_yaml.exists() and repo_yaml.is_file():
             # open repos.yaml file and reat the list of repos
             with repo_yaml.open() as fid:
-                raw = yaml.load(fid, Loader=yaml.Loader)
+                raw = yaml.load(fid)
                 P = raw["repos"]
 
             self._logger.debug(f"the system configuration has a repo file {repo_yaml} refers to {P}")

stackinator/cache.py

@@ -1,15 +1,17 @@
 import os
 import pathlib
 
-import yaml
+from ruamel.yaml import YAML
 
 from . import schema
 
+yaml = YAML()
+
 
 def configuration_from_file(file, mount):
     with file.open() as fid:
         # load the raw yaml input
-        raw = yaml.load(fid, Loader=yaml.Loader)
+        raw = yaml.load(fid)
 
         # validate the yaml
         schema.CacheValidator.validate(raw)

stackinator/recipe.py

@@ -3,11 +3,13 @@
 import re
 
 import jinja2
-import yaml
+from ruamel.yaml import YAML
 
 from . import cache, root_logger, schema, spack_util
 from .etc import envvars
 
+yaml = YAML()
+
 
 class Recipe:
     @property
@@ -58,7 +60,7 @@ def __init__(self, args):
             raise FileNotFoundError(f"The recipe path '{compiler_path}' does not contain compilers.yaml")
 
         with compiler_path.open() as fid:
-            raw = yaml.load(fid, Loader=yaml.Loader)
+            raw = yaml.load(fid)
             schema.CompilersValidator.validate(raw)
             self.generate_compiler_specs(raw)
 
@@ -77,7 +79,7 @@ def __init__(self, args):
         self.packages = None
         if packages_path.is_file():
             with packages_path.open() as fid:
-                self.packages = yaml.load(fid, Loader=yaml.Loader)
+                self.packages = yaml.load(fid)
 
         self._logger.debug("creating packages")
 
@@ -86,7 +88,7 @@ def __init__(self, args):
         recipe_packages_path = self.path / "packages.yaml"
         if recipe_packages_path.is_file():
             with recipe_packages_path.open() as fid:
-                raw = yaml.load(fid, Loader=yaml.Loader)
+                raw = yaml.load(fid)
                 recipe_packages = raw["packages"]
 
         # load system/packages.yaml -> system_packages (if it exists)
@@ -95,7 +97,7 @@ def __init__(self, args):
         if system_packages_path.is_file():
             # load system yaml
             with system_packages_path.open() as fid:
-                raw = yaml.load(fid, Loader=yaml.Loader)
+                raw = yaml.load(fid)
                 system_packages = raw["packages"]
 
         # extract gcc packages from system packages
@@ -115,7 +117,7 @@ def __init__(self, args):
         if network_path.is_file():
             self._logger.debug(f"opening {network_path}")
             with network_path.open() as fid:
-                raw = yaml.load(fid, Loader=yaml.Loader)
+                raw = yaml.load(fid)
                 if "packages" in raw:
                     network_packages = raw["packages"]
                 if "mpi" in raw:
@@ -138,7 +140,7 @@ def __init__(self, args):
             raise FileNotFoundError(f"The recipe path '{environments_path}' does not contain environments.yaml")
 
         with environments_path.open() as fid:
-            raw = yaml.load(fid, Loader=yaml.Loader)
+            raw = yaml.load(fid)
             # add a special environment that installs tools required later in the build process.
             # currently we only need squashfs for creating the squashfs file.
             raw["uenv_tools"] = {
@@ -256,7 +258,7 @@ def config(self, config_path):
             raise FileNotFoundError(f"The recipe path '{config_path}' does not contain config.yaml")
 
         with config_path.open() as fid:
-            raw = yaml.load(fid, Loader=yaml.Loader)
+            raw = yaml.load(fid)
             schema.ConfigValidator.validate(raw)
             self._config = raw
 
@@ -315,7 +317,7 @@ def environment_view_meta(self):
     @property
     def modules_yaml(self):
         with self.modules.open() as fid:
-            raw = yaml.load(fid, Loader=yaml.Loader)
+            raw = yaml.load(fid)
             raw["modules"]["default"]["roots"]["tcl"] = (pathlib.Path(self.mount) / "modules").as_posix()
             return yaml.dump(raw)
 

test_stackinator.py

@@ -5,7 +5,7 @@
 #   "jinja2",
 #   "jsonschema",
 #   "pytest",
-#   "pyYAML",
+#   "ruamel.yaml >=0.15.1, <0.18.0",
 # ]
 # ///
 

unittests/test_schema.py

@@ -5,10 +5,12 @@
 
 import jsonschema
 import pytest
-import yaml
+from ruamel.yaml import YAML
 
 import stackinator.schema as schema
 
+yaml = YAML()
+
 
 @pytest.fixture
 def test_path():
@@ -38,7 +40,7 @@ def recipe_paths(test_path, recipes):
 def test_config_yaml(yaml_path):
     # test that the defaults are set as expected
     with open(yaml_path / "config.defaults.yaml") as fid:
-        raw = yaml.load(fid, Loader=yaml.Loader)
+        raw = yaml.load(fid)
         schema.ConfigValidator.validate(raw)
         assert raw["store"] == "/user-environment"
         assert raw["spack"]["commit"] is None
@@ -57,10 +59,7 @@ def test_config_yaml(yaml_path):
             repo: https://github.com/spack/spack.git
             commit: develop-packages
     """)
-    raw = yaml.load(
-        config,
-        Loader=yaml.Loader,
-    )
+    raw = yaml.load(config)
     schema.ConfigValidator.validate(raw)
     assert raw["spack"]["commit"] is None
     assert raw["spack"]["packages"]["commit"] is not None
@@ -78,10 +77,7 @@ def test_config_yaml(yaml_path):
         packages:
             repo: https://github.com/spack/spack.git
     """)
-    raw = yaml.load(
-        config,
-        Loader=yaml.Loader,
-    )
+    raw = yaml.load(config)
     schema.ConfigValidator.validate(raw)
     assert raw["spack"]["commit"] == "develop"
     assert raw["spack"]["packages"]["commit"] is None
@@ -91,7 +87,7 @@ def test_config_yaml(yaml_path):
 
     # full config
     with open(yaml_path / "config.full.yaml") as fid:
-        raw = yaml.load(fid, Loader=yaml.Loader)
+        raw = yaml.load(fid)
         schema.ConfigValidator.validate(raw)
         assert raw["store"] == "/alternative-point"
         assert raw["spack"]["commit"] == "6408b51"
@@ -107,28 +103,28 @@ def test_config_yaml(yaml_path):
         spack:
             repo: https://github.com/spack/spack.git
         """)
-        raw = yaml.load(config, Loader=yaml.Loader)
+        raw = yaml.load(config)
         schema.ConfigValidator.validate(raw)
 
 
 def test_recipe_config_yaml(recipe_paths):
     # validate the config.yaml in the test recipes
     for p in recipe_paths:
         with open(p / "config.yaml") as fid:
-            raw = yaml.load(fid, Loader=yaml.Loader)
+            raw = yaml.load(fid)
             schema.ConfigValidator.validate(raw)
 
 
 def test_compilers_yaml(yaml_path):
     # test that the defaults are set as expected
     with open(yaml_path / "compilers.defaults.yaml") as fid:
-        raw = yaml.load(fid, Loader=yaml.Loader)
+        raw = yaml.load(fid)
         schema.CompilersValidator.validate(raw)
         assert raw["gcc"] == {"version": "10.2"}
         assert raw["llvm"] is None
 
     with open(yaml_path / "compilers.full.yaml") as fid:
-        raw = yaml.load(fid, Loader=yaml.Loader)
+        raw = yaml.load(fid)
         schema.CompilersValidator.validate(raw)
         assert raw["gcc"] == {"version": "11"}
         assert raw["llvm"] == {"version": "13"}
@@ -139,13 +135,13 @@ def test_recipe_compilers_yaml(recipe_paths):
     # validate the compilers.yaml in the test recipes
     for p in recipe_paths:
         with open(p / "compilers.yaml") as fid:
-            raw = yaml.load(fid, Loader=yaml.Loader)
+            raw = yaml.load(fid)
             schema.CompilersValidator.validate(raw)
 
 
 def test_environments_yaml(yaml_path):
     with open(yaml_path / "environments.full.yaml") as fid:
-        raw = yaml.load(fid, Loader=yaml.Loader)
+        raw = yaml.load(fid)
         schema.EnvironmentsValidator.validate(raw)
 
         # the defaults-env does not set fields
@@ -186,7 +182,7 @@ def test_environments_yaml(yaml_path):
     # check that only allowed fields are accepted
     # from an example that was silently validated
     with open(yaml_path / "environments.err-providers.yaml") as fid:
-        raw = yaml.load(fid, Loader=yaml.Loader)
+        raw = yaml.load(fid)
         with pytest.raises(
             jsonschema.exceptions.ValidationError,
             match=r"Additional properties are not allowed \('providers' was unexpected",
@@ -198,7 +194,7 @@ def test_recipe_environments_yaml(recipe_paths):
     # validate the environments.yaml in the test recipes
     for p in recipe_paths:
         with open(p / "environments.yaml") as fid:
-            raw = yaml.load(fid, Loader=yaml.Loader)
+            raw = yaml.load(fid)
             schema.EnvironmentsValidator.validate(raw)
 
 
@@ -210,5 +206,7 @@ def test_unique_properties():
         """
     )
 
-    with pytest.raises(Exception):
-        yaml.load(invalid_config, Loader=yaml.Loader)
+    from ruamel.yaml.constructor import DuplicateKeyError
+
+    with pytest.raises(DuplicateKeyError):
+        yaml.load(invalid_config)