Skip module in the symlinks (#511)

* Implemented "default_version" for TCL
* Use write_version_file for the symlink tree too
* Skip `module.tcl` in the symlinks

This is done by symlinking `<software>/<version>` itself to
`<namespace>/<software>/<version>/module.tcl`.
For the directory of the wrapper scripts to be correctly found, the
symlink has to be resolved, but TCL's `file normalize` won't normalise
the filename. So, we need to use `file readlink` instead, but only on
real symlinks because it raises an error.

* Symlink to module.lua when possible
which is when default_version==True (default_version==False can't be
made to work with symlinks).

* Added a `--force` option to `shpc install` to force overwriting existing symlinks
The name `--force` is generic, so that other things could be forced
through it, not just overwriting symlinks.
Also added an info message if a symlink is overwritten, which can be
hidden with the `--quiet` flag.

* Made `force` optional
* Forgot the variable for substitution
* The "delete" command was superseded by "uninstall" in #6
* Added `--no-symlink-tree` to override the config file
`--symlink-tree` now also overrides the config file
* Make it explicit we are expecting yes or no

Author: muffato

docs/getting_started/user-guide.rst

@@ -189,7 +189,7 @@ variable replacement. A summary table of variables is included below, and then f
      - a timestamp to keep track of when you last saved
      - never
    * - default_version
-     - A boolean to indicate generating a .version file (LMOD or lua modules only)
+     - A boolean to indicate whether a default version will be arbitrarily chosen, when multiple versions are available, and none is explicitly requested
      - true
    * - singularity_module
      - if defined, add to module script to load this Singularity module first

shpc/client/__init__.py

@@ -103,7 +103,21 @@ def get_parser():
     install.add_argument(
         "--symlink-tree",
         dest="symlink",
-        help="install to symlink tree too.",
+        help="install to symlink tree too (overrides settings.yml).",
+        default=None,
+        action="store_true",
+    )
+    install.add_argument(
+        "--no-symlink-tree",
+        dest="symlink",
+        help="skip installing to symlink tree (in case set in settings.yml).",
+        action="store_false",
+    )
+    install.add_argument(
+        "--force",
+        "-f",
+        dest="force",
+        help="replace existing symlinks",
         default=False,
         action="store_true",
     )
@@ -362,8 +376,6 @@ def help(return_code=0):
         from .docgen import main
     elif args.command == "get":
         from .get import main
-    elif args.command == "delete":
-        from .delete import main
     elif args.command == "install":
         from .install import main
     elif args.command == "inspect":

shpc/client/install.py

@@ -21,4 +21,4 @@ def main(args, parser, extra, subparser):
     cli.settings.update_params(args.config_params)
 
     # And do the install
-    cli.install(args.install_recipe, symlink=args.symlink)
+    cli.install(args.install_recipe, symlink=args.symlink, force=args.force)

shpc/main/modules/__init__.py

@@ -11,7 +11,6 @@
 
 from datetime import datetime
 import os
-from pathlib import Path
 import shutil
 import subprocess
 import sys
@@ -114,6 +113,7 @@ def _uninstall(self, module_dir, name, force=False):
                 msg = "%s, and all content below it? " % name
                 if not utils.confirm_uninstall(msg, force):
                     return
+            self._cleanup_symlink(module_dir)
             self._cleanup(module_dir)
             logger.info("%s and all subdirectories have been removed." % name)
         else:
@@ -183,7 +183,15 @@ def get_symlink_path(self, module_dir):
         """
         if not self.settings.symlink_base:
             return
-        return os.path.join(self.settings.symlink_base, *module_dir.split(os.sep)[-2:])
+
+        symlink_base_name = os.path.join(self.settings.symlink_base, *module_dir.split(os.sep)[-2:])
+
+        # With Lmod and default_version==True, the symlinks points to module.lua itself,
+        # and its name needs to end with `.lua` too
+        if self.module_extension == "lua" and self.settings.default_version == True:
+            return symlink_base_name + ".lua"
+        else:
+            return symlink_base_name
 
     def create_symlink(self, module_dir):
         """
@@ -192,33 +200,39 @@ def create_symlink(self, module_dir):
         symlink_path = self.get_symlink_path(module_dir)
         if os.path.exists(symlink_path):
             os.unlink(symlink_path)
-        logger.info("Creating link %s -> %s" % (module_dir, symlink_path))
         symlink_dir = os.path.dirname(symlink_path)
 
         # If the parent directory doesn't exist, make it
         if not os.path.exists(symlink_dir):
             utils.mkdirp([symlink_dir])
 
+        # With Lmod, default_version==False can't be made to work with symlinks at the module.lua level
+        if self.module_extension == "lua" and self.settings.default_version == False:
+            symlink_target = module_dir
+        else:
+            symlink_target = os.path.join(module_dir, self.modulefile)
+        logger.info("Creating link %s -> %s" % (symlink_target, symlink_path))
+
         # Create the symbolic link!
-        os.symlink(module_dir, symlink_path)
+        os.symlink(symlink_target, symlink_path)
 
-        # If we don't have a version file in root, create it
-        if self.module_extension != "tcl" and self.settings.default_version == True:
-            version_file = os.path.join(os.path.dirname(symlink_path), ".version")
-            if not os.path.exists(version_file):
-                Path(version_file).touch()
+        # Create .version
+        self.write_version_file(os.path.dirname(symlink_path))
 
-    def check_symlink(self, module_dir):
+    def check_symlink(self, module_dir, force=False):
         """
         Given an install command, if --symlink-tree is provided make
         sure we don't already have this symlink in the tree.
         """
         # Get the symlink path - does it exist?
         symlink_path = self.get_symlink_path(module_dir)
-        if os.path.exists(symlink_path) and not utils.confirm_action(
-            "%s already exists, are you sure you want to overwrite?" % symlink_path
-        ):
-            sys.exit(0)
+        if os.path.exists(symlink_path):
+            if force:
+                logger.info("Overwriting %s, as requested" % module_dir)
+            elif not utils.confirm_action(
+                "%s already exists, are you sure you want to overwrite" % symlink_path
+            ):
+                sys.exit(0)
 
     def _cleanup_symlink(self, module_dir):
         """
@@ -343,7 +357,24 @@ def check(self, module_name):
         config = self._load_container(module_name.rsplit(":", 1)[0])
         return self.container.check(module_name, config)
 
-    def install(self, name, tag=None, symlink=False, **kwargs):
+    def write_version_file(self, version_dir):
+        """
+        Create the .version file, if there is a template for it.
+
+        Note that we don't actually change the content of the template:
+        it is copied as is.
+        """
+        version_template = 'default_version.' + self.module_extension
+        if not self.settings.default_version:
+            version_template = 'no_' + version_template
+        template_file = os.path.join(here, "templates", version_template)
+        if os.path.exists(template_file):
+            version_file = os.path.join(version_dir, ".version")
+            if not os.path.exists(version_file):
+                version_content = shpc.utils.read_file(template_file)
+                shpc.utils.write_file(version_file, version_content)
+
+    def install(self, name, tag=None, symlink=None, force=False, **kwargs):
         """
         Given a unique resource identifier, install a recipe.
 
@@ -372,20 +403,18 @@ def install(self, name, tag=None, symlink=False, **kwargs):
         subfolder = os.path.join(uri, tag.name)
         container_dir = self.container.container_dir(subfolder)
 
-        # Global override to arg
-        symlink = self.settings.symlink_tree is True or symlink
+        # Default to global setting
+        if symlink is None:
+            symlink = self.settings.symlink_tree
 
         if symlink:
             # Cut out early if symlink desired and already exists
-            self.check_symlink(module_dir)
+            self.check_symlink(module_dir, force)
         shpc.utils.mkdirp([module_dir, container_dir])
 
         # Add a .version file to indicate the level of versioning (not for tcl)
-        if self.module_extension != "tcl" and self.settings.default_version == True:
-            version_dir = os.path.join(self.settings.module_base, uri)
-            version_file = os.path.join(version_dir, ".version")
-            if not os.path.exists(version_file):
-                Path(version_file).touch()
+        version_dir = os.path.join(self.settings.module_base, uri)
+        self.write_version_file(version_dir)
 
         # For Singularity this is a path, podman is a uri. If None is returned
         # there was an error and we cleanup

shpc/main/modules/templates/default_version.lua

@@ -40,8 +40,8 @@ For each of the above, you can export:
 if not os.getenv("PODMAN_OPTS") then setenv ("PODMAN_OPTS", "") end
 if not os.getenv("PODMAN_COMMAND_OPTS") then setenv ("PODMAN_COMMAND_OPTS", "") end
 
--- directory containing this modulefile (dynamically defined)
-local moduleDir = myFileName():match("(.*[/])") or "."
+-- directory containing this modulefile, once symlinks resolved (dynamically defined)
+local moduleDir = subprocess("realpath " .. myFileName()):match("(.*[/])") or "."
 
 -- interactive shell to any container, plus exec for aliases
 local containerPath = '{{ image }}'

shpc/main/modules/templates/docker.lua

@@ -55,8 +55,8 @@ set helpcommand "This module is a {{ docker }} container wrapper for {{ name }}
 {% if labels %}{% for key, value in labels.items() %}set {{ key }} "{{ value }}"
 {% endfor %}{% endif %}
 
-# directory containing this modulefile (dynamically defined)
-set moduleDir   "[file dirname ${ModulesCurrentModulefile}]"
+# directory containing this modulefile, once symlinks resolved (dynamically defined)
+set moduleDir   [file dirname [expr { [string equal [file type ${ModulesCurrentModulefile}] "link"] ? [file readlink ${ModulesCurrentModulefile}] : ${ModulesCurrentModulefile} }]]
 
 # conflict with modules with the same alias name
 conflict {{ parsed_name.tool }}

shpc/main/modules/templates/docker.tcl

@@ -0,0 +1,2 @@
+#%Module
+set ModulesVersion "please_specify_a_version_number"

shpc/main/modules/templates/no_default_version.tcl

@@ -38,8 +38,8 @@ For each of the above, you can export:
 
 {% if settings.singularity_module %}load("{{ settings.singularity_module }}"){% endif %}
 
--- directory containing this modulefile (dynamically defined)
-local moduleDir = myFileName():match("(.*[/])") or "."
+-- directory containing this modulefile, once symlinks resolved (dynamically defined)
+local moduleDir = subprocess("realpath " .. myFileName()):match("(.*[/])") or "."
 
 -- singularity environment variable to set shell
 setenv("SINGULARITY_SHELL", "{{ settings.singularity_shell }}")

shpc/main/modules/templates/singularity.lua

@@ -59,8 +59,8 @@ set helpcommand "This module is a singularity container wrapper for {{ name }} v
 {% if labels %}{% for key, value in labels.items() %}set {{ key }} "{{ value }}"
 {% endfor %}{% endif %}
 
-# directory containing this modulefile (dynamically defined)
-set moduleDir   "[file dirname ${ModulesCurrentModulefile}]"
+# directory containing this modulefile, once symlinks resolved (dynamically defined)
+set moduleDir   [file dirname [expr { [string equal [file type ${ModulesCurrentModulefile}] "link"] ? [file readlink ${ModulesCurrentModulefile}] : ${ModulesCurrentModulefile} }]]
 
 # conflict with modules with the same alias name
 conflict {{ parsed_name.tool }}

shpc/main/modules/templates/singularity.tcl

@@ -27,7 +27,7 @@ module_base: $root_dir/modules
 # This is where you might add a prefix to your module names, if desired.
 module_name: '{{ parsed_name.tool }}'
 
-# Create a .version file for LMOD in the module folder
+# When multiple versions are available and none requested, allow module picking one iself
 default_version: true
 
 # store containers separately from module files

shpc/settings.yml

@@ -95,7 +95,7 @@ def confirm_action(question, force=False):
     if force is True:
         return True
 
-    response = input(question)
+    response = input(question + " (yes/no)?")
     while len(response) < 1 or response[0].lower().strip() not in "ynyesno":
         response = input("Please answer yes or no: ")