diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8ce21ca2a..059ae429b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -27,7 +27,7 @@ jobs: run: | export PATH="/usr/share/miniconda/bin:$PATH" source activate black - pip install black==20.8b1 + pip install black black --check shpc - name: Check imports with pyflakes diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 24bb424ed..4924ace7c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -100,8 +100,12 @@ jobs: cat test_output grep --quiet 'Python 3.9.5' test_output rm test_output - shpc uninstall --force python:3.9.5-alpine + shpc uninstall --force python:3.9.5-alpine + # Try creating symlink install + mkdir -p tmp-modules + shpc config set symlink_base:tmp-modules + shpc install python:3.9.5-alpine --symlink-tree - name: Run python module tests (tcsh) shell: tcsh -e {0} diff --git a/CHANGELOG.md b/CHANGELOG.md index c804bceae..7ee63273a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ and **Merged pull requests**. Critical items to know are: The versions coincide with releases on pip. Only major versions will be released as tags on Github. ## [0.0.x](https://github.com/singularityhub/singularity-hpc/tree/main) (0.0.x) + - support for installing to symlink tree (0.0.52) + - also including cleanup of symlink tree on uninstall + - ability to set custom config variable on the fly with -c - minimum version of spython required is 0.2.0 to support apptainer (0.0.51) - add support for TCL and LMOD default version, multiple variants (0.0.50) - refactor to "add" to generate a container.yaml first (0.0.49) diff --git a/docs/getting_started/user-guide.rst b/docs/getting_started/user-guide.rst index 90a267001..95fe37aaf 100644 --- a/docs/getting_started/user-guide.rst +++ b/docs/getting_started/user-guide.rst @@ -52,6 +52,11 @@ You can then easily install, load, and use modules: $ module load biocontainers/samtools $ samtools +Or set a configuration value on the fly for any command: + +.. code-block:: console + + $ shpc install -c set:symlink_base:/tmp/modules biocontainers/samtools The above assumes that you've installed the software, and have already added the modules folder to be seen by your module software. If your module @@ -174,6 +179,12 @@ variable replacement. A summary table of variables is included below, and then f * - container_tech - The container technology to use (singularity or podman) - singularity + * - symlink_base + - If set, where you want to install a simplified module tree to using the ``symlink-tree`` option + - $root_dir/symlinks + * - symlink_tree + - If set to true, always generate a symlink unless the ``--no-symlink-tree`` flag is given + - false * - updated_at - a timestamp to keep track of when you last saved - never @@ -233,6 +244,14 @@ variable replacement. A summary table of variables is included below, and then f - All features default to null +Note that any configuration value can be set permanently by using ``shpc config`` +or manually editing the file, but you can also set config values "one off" as follows: + +.. code-block:: console + + $ shpc install -c set:symlink_base:/tmp/modules ghcr.io/autamus/clingo + + These settings will be discussed in more detail in the following sections. Features @@ -359,6 +378,58 @@ you can add or remove entries via the config variable ``registry`` # Note that "add" is used for lists of things (e.g., the registry config variable is a list) and "set" is used to set a key value pair. +Symlink Base +------------ + +By default, your modules are installed to your ``module_base`` described above with a complete +namespace, meaning the container registry from where they arise. We do this so that the namespace +is consistent and there are no conflicts. However, if you want a simplified tree to install from, +meaning the module full names are _just_ the final container name, you can set the ``symlink_base`` +in your settings to a different root. For example, let's say we want to install a set of modules. +We can use the default ``symlink_base`` of ``$root_dir/symlinks`` or set our own ``symlink_base`` +in the settings.yaml. We could do: + +.. code-block:: console + + $ shpc install ghcr.io/autamus/clingo --symlink-tree + $ shpc install ghcr.io/autamus/samtools --symlink-tree + +Then, for example, if you want to load the modules, you'll see the shorter names are +available! + +.. code-block:: console + + $ module use ./symlinks + $ module load clingo/5.5.1 + +This is much more efficient compared to the install that uses the full paths: + +.. code-block:: console + + $ module use ./modules + $ module load ghcr.io/autamus/clingo/5.5.1/module + +Since we install based on the container name *and* version tag, this even gives you +the ability to install versions from different container bases in the same root. +If there is a conflict, you will be given the option to exit (and abort) or continue. +Finally, if you need an easy way to run through the containers you've already installed +to create the links: + + +.. code-block:: console + + for module in $(shpc list); do + shpc install $module --symlink-tree + done + +And that will reinstall the modules you have installed, but in their symlink tree location. + + +.. warning:: + + Be cautious about creating symlinks in containers or other contexts where a bind + could eliminate the symlink or make the path non-existent. + Default Version --------------- diff --git a/shpc/client/__init__.py b/shpc/client/__init__.py index 9e494924e..6183de682 100644 --- a/shpc/client/__init__.py +++ b/shpc/client/__init__.py @@ -40,6 +40,17 @@ def get_parser(): help="custom path to settings file.", ) + # On the fly updates to config params + parser.add_argument( + "-c", + dest="config_params", + help=""""customize a config value on the fly to ADD/SET/REMOVE for a command +shpc -c set:key:value +shpc -c add:registry:/tmp/registry +shpc -c rm:registry:/tmp/registry""", + action="append", + ) + parser.add_argument( "--version", dest="version", @@ -89,6 +100,27 @@ def get_parser(): "install_recipe", help="recipe to install\nshpc install python\nshpc install python:3.9.5-alpine", ) + install.add_argument( + "--symlink-tree", + dest="symlink", + help="install to symlink tree too (overrides settings.yml).", + default=None, + action="store_true", + ) + install.add_argument( + "--no-symlink-tree", + dest="symlink", + help="skip installing to symlink tree (in case set in settings.yml).", + action="store_false", + ) + install.add_argument( + "--force", + "-f", + dest="force", + help="replace existing symlinks", + default=False, + action="store_true", + ) # List installed modules listing = subparsers.add_parser("list", description="list installed modules.") @@ -149,7 +181,6 @@ def get_parser(): config.add_argument( "--central", - "-c", dest="central", help="make edits to the central config file.", default=False, @@ -347,8 +378,6 @@ def help(return_code=0): from .docgen import main elif args.command == "get": from .get import main - elif args.command == "delete": - from .delete import main elif args.command == "install": from .install import main elif args.command == "inspect": diff --git a/shpc/client/add.py b/shpc/client/add.py index 1f73e2bfb..8901b6a61 100644 --- a/shpc/client/add.py +++ b/shpc/client/add.py @@ -13,5 +13,9 @@ def main(args, parser, extra, subparser): module=args.module, container_tech=args.container_tech, ) + + # Update config settings on the fly + cli.settings.update_params(args.config_params) + # If we don't have a module name, we derive from container URI cli.add(args.container_uri, args.module_id) diff --git a/shpc/client/check.py b/shpc/client/check.py index 07b76fad7..1998c0c87 100644 --- a/shpc/client/check.py +++ b/shpc/client/check.py @@ -13,4 +13,7 @@ def main(args, parser, extra, subparser): module=args.module, container_tech=args.container_tech, ) + + # Update config settings on the fly + cli.settings.update_params(args.config_params) cli.check(args.module_name) diff --git a/shpc/client/config.py b/shpc/client/config.py index 86dc97fe8..ad224a131 100644 --- a/shpc/client/config.py +++ b/shpc/client/config.py @@ -33,25 +33,12 @@ def main(args, parser, extra, subparser): return cli.settings.inituser() if command == "edit": return cli.settings.edit() - elif command in ["set", "add", "remove"]: - for param in args.params: - if ":" not in param: - logger.warning( - "Param %s is missing a :, should be key:value pair. Skipping." - % param - ) - continue - key, value = param.split(":", 1) - if command == "set": - cli.settings.set(key, value) - logger.info("Updated %s to be %s" % (key, value)) - elif command == "add": - cli.settings.add(key, value) - logger.info("Added %s to %s" % (key, value)) - elif command == "remove": - cli.settings.remove(key, value) - logger.info("Removed %s from %s" % (key, value)) + if command in ["set", "add", "remove"]: + + # Update each param + for param in args.params: + cli.settings.update_param(command, param) # Save settings cli.settings.save() diff --git a/shpc/client/docgen.py b/shpc/client/docgen.py index 10e54a4bf..7c6ecb994 100644 --- a/shpc/client/docgen.py +++ b/shpc/client/docgen.py @@ -13,4 +13,6 @@ def main(args, parser, extra, subparser): module=args.module, container_tech=args.container_tech, ) + # Update config settings on the fly + cli.settings.update_params(args.config_params) cli.docgen(args.module_name) diff --git a/shpc/client/get.py b/shpc/client/get.py index 47cfde176..a600aec87 100644 --- a/shpc/client/get.py +++ b/shpc/client/get.py @@ -14,6 +14,8 @@ def main(args, parser, extra, subparser): container_tech=args.container_tech, ) + # Update config settings on the fly + cli.settings.update_params(args.config_params) result = cli.get(args.module_name, args.env_file) if result: print(result) diff --git a/shpc/client/inspect.py b/shpc/client/inspect.py index 372c84cec..15999d539 100644 --- a/shpc/client/inspect.py +++ b/shpc/client/inspect.py @@ -15,6 +15,9 @@ def main(args, parser, extra, subparser): settings_file=args.settings_file, container_tech=args.container_tech, ) + + # Update config settings on the fly + cli.settings.update_params(args.config_params) metadata = cli.inspect(args.module_name) # Case 1: dump entire thing as json diff --git a/shpc/client/install.py b/shpc/client/install.py index 2b7cefc4d..07e0b7398 100644 --- a/shpc/client/install.py +++ b/shpc/client/install.py @@ -16,4 +16,9 @@ def main(args, parser, extra, subparser): module=args.module, container_tech=args.container_tech, ) - cli.install(args.install_recipe) + + # Update config settings on the fly + cli.settings.update_params(args.config_params) + + # And do the install + cli.install(args.install_recipe, symlink=args.symlink, force=args.force) diff --git a/shpc/client/listing.py b/shpc/client/listing.py index c00fd5358..29d6e097c 100644 --- a/shpc/client/listing.py +++ b/shpc/client/listing.py @@ -14,4 +14,6 @@ def main(args, parser, extra, subparser): container_tech=args.container_tech, ) + # Update config settings on the fly + cli.settings.update_params(args.config_params) cli.list(args.pattern, args.names_only, short=args.short) diff --git a/shpc/client/namespace.py b/shpc/client/namespace.py index 79db0867a..aab992a39 100644 --- a/shpc/client/namespace.py +++ b/shpc/client/namespace.py @@ -13,6 +13,9 @@ def main(args, parser, extra, subparser): cli = get_client(quiet=args.quiet, settings_file=args.settings_file) + # Update config settings on the fly + cli.settings.update_params(args.config_params) + # Case 1: we need to unset a namespace if not args.namespace: sys.exit("Please choose: shpc use or shpc unset.") diff --git a/shpc/client/shell.py b/shpc/client/shell.py index 2fcb3182c..b40f36581 100644 --- a/shpc/client/shell.py +++ b/shpc/client/shell.py @@ -34,13 +34,17 @@ def main(args, parser, extra, subparser): def create_client(args): from shpc.main import get_client - return get_client( + cli = get_client( quiet=args.quiet, settings_file=args.settings_file, module=args.module, container_tech=args.container_tech, ) + # Update config settings on the fly + cli.settings.update_params(args.config_params) + return cli + def ipython(args): """ diff --git a/shpc/client/show.py b/shpc/client/show.py index 716168206..23a7baa70 100644 --- a/shpc/client/show.py +++ b/shpc/client/show.py @@ -8,4 +8,7 @@ def main(args, parser, extra, subparser): from shpc.main import get_client cli = get_client(quiet=args.quiet, settings_file=args.settings_file) + + # Update config settings on the fly + cli.settings.update_params(args.config_params) cli.show(args.name, names_only=not args.versions, filter_string=args.filter_string) diff --git a/shpc/client/test.py b/shpc/client/test.py index 26dca8d82..31d9402b7 100644 --- a/shpc/client/test.py +++ b/shpc/client/test.py @@ -14,6 +14,8 @@ def main(args, parser, extra, subparser): container_tech=args.container_tech, ) + # Update config settings on the fly + cli.settings.update_params(args.config_params) cli.test( args.module_name, test_exec=args.test_exec, diff --git a/shpc/client/uninstall.py b/shpc/client/uninstall.py index b599cef19..be40d1987 100644 --- a/shpc/client/uninstall.py +++ b/shpc/client/uninstall.py @@ -13,4 +13,7 @@ def main(args, parser, extra, subparser): module=args.module, container_tech=args.container_tech, ) + + # Update config settings on the fly + cli.settings.update_params(args.config_params) cli.uninstall(args.uninstall_recipe, args.force) diff --git a/shpc/defaults.py b/shpc/defaults.py index 90a640c39..4424d45c8 100644 --- a/shpc/defaults.py +++ b/shpc/defaults.py @@ -17,7 +17,7 @@ ) # variables in settings that allow environment variable expansion -allowed_envars = ["container_base", "module_base", "registry"] +allowed_envars = ["container_base", "module_base", "symlink_base", "registry"] # The GitHub repository with recipes github_url = "https://github.com/singularityhub/singularity-hpc" diff --git a/shpc/main/modules/base.py b/shpc/main/modules/base.py index c24c11edd..57f22345a 100644 --- a/shpc/main/modules/base.py +++ b/shpc/main/modules/base.py @@ -8,7 +8,6 @@ import shpc.defaults as defaults import shpc.main.templates import shpc.main.container as container -from jinja2 import Template from datetime import datetime import os @@ -17,7 +16,13 @@ import sys import inspect -here = os.path.abspath(os.path.dirname(__file__)) +from jinja2 import Environment, FileSystemLoader + +here = os.path.dirname(os.path.abspath(__file__)) + +# Allow includes from this directory OR providing strings +template_dir = os.path.join(here, "templates") +env = Environment(loader=FileSystemLoader(template_dir)) class ModuleBase(BaseClient): @@ -42,9 +47,8 @@ def _load_template(self, template_name): """ template_file = self._get_template(template_name) - # Make all substitutions here with open(template_file, "r") as temp: - template = Template(self.substitute(temp.read())) + template = env.from_string(self.substitute(temp.read())) return template def substitute(self, template): @@ -87,14 +91,15 @@ def uninstall(self, name, force=False): module_dir = os.path.join(self.settings.module_base, name) container_dir = self.container.container_dir(name) - # Podman needs image deletion - self.container.delete(name) - + # Ask before deleting anything! if not force: msg = name + "?" if not utils.confirm_uninstall(msg, force): return + # Podman needs image deletion + self.container.delete(name) + if container_dir != module_dir: self._uninstall( container_dir, self.container_base, "$container_base/%s" % name @@ -107,6 +112,9 @@ def uninstall(self, name, force=False): module_dir, self.settings.module_base, "$module_base/%s" % name ) + # Clean up symbolic links + self._cleanup_symlink(module_dir) + # parent of versioned directory has module .version module_dir = os.path.dirname(module_dir) @@ -119,7 +127,7 @@ def _uninstall(self, path, base_path, name): Sub function, so we can pass more than one folder from uninstall """ if os.path.exists(path): - utils.rmdir_to_base(path, base_path) + utils.remove_to_base(path, base_path) logger.info("%s and all subdirectories have been removed." % name) else: logger.warning("%s does not exist." % name) @@ -197,6 +205,81 @@ def list(self, pattern=None, names_only=False, out=None, short=False): short=short, ) + # Symbolic links + + def get_symlink_path(self, module_dir): + """ + Should only be called given a self.settings.symlink_base is set + """ + if not self.settings.symlink_base: + return + + symlink_base_name = os.path.join( + self.settings.symlink_base, *module_dir.split(os.sep)[-2:] + ) + + return symlink_base_name + self.symlink_extension + + def create_symlink(self, module_dir): + """ + Create the symlink if desired by the user! + """ + symlink_path = self.get_symlink_path(module_dir) + if os.path.exists(symlink_path): + os.unlink(symlink_path) + symlink_dir = os.path.dirname(symlink_path) + + # If the parent directory doesn't exist, make it + if not os.path.exists(symlink_dir): + utils.mkdirp([symlink_dir]) + + symlink_target = os.path.join(module_dir, self.modulefile) + logger.info("Creating link %s -> %s" % (symlink_target, symlink_path)) + + # Create the symbolic link! + os.symlink(symlink_target, symlink_path) + + # Create .version + self.write_version_file(os.path.dirname(symlink_path)) + + def check_symlink(self, module_dir, force=False): + """ + Given an install command, if --symlink-tree is provided make + sure we don't already have this symlink in the tree. + """ + # Get the symlink path - does it exist? + symlink_path = self.get_symlink_path(module_dir) + if not symlink_path: + logger.exit( + "symlink_base is not set, cannot create a symlink without it. Check your settings." + ) + elif os.path.exists(symlink_path): + if force: + logger.info("Overwriting %s, as requested" % module_dir) + elif not utils.confirm_action( + "%s already exists, are you sure you want to overwrite" % symlink_path + ): + sys.exit(0) + + def _cleanup_symlink(self, module_dir): + """ + Remove symlink directories if they exist + """ + symlinked_module = self.get_symlink_path(module_dir) + if not symlinked_module: + return + if os.path.islink(symlinked_module): + # Remove and clean up directories that become empty + utils.remove_to_base(symlinked_module, self.settings.symlink_base) + logger.info("%s has been removed." % symlinked_module) + # Update .version + self.write_version_file(os.path.dirname(symlinked_module)) + elif os.path.exists(symlinked_module): + logger.error("%s exists and is not a symlink!" % symlinked_module) + elif self.settings.symlink_tree: + # Should not happen. The symlink has someone already been deleted + logger.warning("%s does not exist." % symlinked_module) + def docgen(self, module_name, out=None): """ Render documentation for a module. @@ -300,7 +383,7 @@ def check(self, module_name): config = self._load_container(module_name.rsplit(":", 1)[0]) return self.container.check(module_name, config) - def install(self, name, tag=None, **kwargs): + def install(self, name, tag=None, symlink=None, force=False, **kwargs): """ Given a unique resource identifier, install a recipe. @@ -332,6 +415,15 @@ def install(self, name, tag=None, **kwargs): module_dir = os.path.join(self.settings.module_base, uri, tag.name) subfolder = os.path.join(uri, tag.name) container_dir = self.container.container_dir(subfolder) + + # Default to global setting + if symlink is None: + symlink = self.settings.symlink_tree + + if symlink: + # Cut out early if symlink desired and already exists + self.check_symlink(module_dir, force=force) + shpc.utils.mkdirp([module_dir, container_dir]) # If we have a sif URI provided by path, the container needs to exist @@ -353,7 +445,7 @@ def install(self, name, tag=None, **kwargs): container_path = container_dest # Add a .version file to indicate the level of versioning - self.write_version_file(uri, tag.name) + self.write_version_file(os.path.join(self.settings.module_base, uri), tag.name) # For Singularity this is a path, podman is a uri. If None is returned # there was an error and we cleanup @@ -362,7 +454,7 @@ def install(self, name, tag=None, **kwargs): module_dir, container_dir, config, tag ) if not container_path: - utils.rmdir_to_base(container_dir, self.container_base) + utils.remove_to_base(container_dir, self.container_base) logger.exit("There was an issue pulling %s" % container_path) # Get the template based on the module and container type @@ -392,7 +484,7 @@ def install(self, name, tag=None, **kwargs): # If the container tech does not need storage, clean up if not os.listdir(container_dir): - utils.rmdir_to_base(container_dir, self.container_base) + utils.remove_to_base(container_dir, self.container_base) # Write the environment file to be bound to the container self.container.add_environment( @@ -404,6 +496,10 @@ def install(self, name, tag=None, **kwargs): if ":" not in name: name = "%s:%s" % (name, tag.name) logger.info("Module %s was created." % name) + + if symlink: + self.create_symlink(module_dir) + return container_path # Module software can choose how to handle each of these cases @@ -420,19 +516,22 @@ def _set_default_version(self, version_file, tag): template = self._load_template("default_version") utils.write_file(version_file, template.render(version=tag)) - def write_version_file(self, uri, latest_tag_installed=None): + def write_version_file(self, version_dir, latest_tag_installed=None): """ Create the .version file, if there is a template for it. """ - version_dir = os.path.join(self.settings.module_base, uri) + if not os.path.exists(version_dir): + # Happens when uninstalling the last version of a tool + return + version_file = os.path.join(version_dir, ".version") # No default versions - if self.settings.default_version in [False, None]: + if not self.settings.default_version: return self._no_default_version(version_file, latest_tag_installed) # allow the module software to control versions - if self.settings.default_version in [True, "module_sys"]: + if self.settings.default_version == "module_sys": return self._module_sys_default_version(version_file, latest_tag_installed) # First or last installed diff --git a/shpc/main/modules/lmod.py b/shpc/main/modules/lmod.py index 1bd16a31a..967707658 100644 --- a/shpc/main/modules/lmod.py +++ b/shpc/main/modules/lmod.py @@ -13,6 +13,8 @@ def __init__(self, **kwargs): """ super(Client, self).__init__(**kwargs) self.module_extension = "lua" + # With Lmod, the symlink names must end with `.lua` too + self.symlink_extension = ".lua" def _module_sys_default_version(self, version_file, tag=None): """ diff --git a/shpc/main/modules/tcl.py b/shpc/main/modules/tcl.py index a479fb9bf..043331bae 100644 --- a/shpc/main/modules/tcl.py +++ b/shpc/main/modules/tcl.py @@ -12,7 +12,10 @@ def __init__(self, **kwargs): An Lmod client generates an lmod recipe for install """ super(Client, self).__init__(**kwargs) + # The extension is technically not required, but we still set one for clarity self.module_extension = "tcl" + # Except for symlink, since the goal is to have short names + self.symlink_extension = "" def _no_default_version(self, version_file, tag=None): """ diff --git a/shpc/main/modules/templates/docker.lua b/shpc/main/modules/templates/docker.lua index 780311678..ae5471016 100644 --- a/shpc/main/modules/templates/docker.lua +++ b/shpc/main/modules/templates/docker.lua @@ -34,14 +34,15 @@ For each of the above, you can export: - PODMAN_COMMAND_OPTS: to define custom options for the command ]]) +{% include "includes/default_version.lua" %} {% if settings.podman_module %}load("{{ settings.podman_module }}"){% endif %} -- Environment: only set options and command options if not already set if not os.getenv("PODMAN_OPTS") then setenv ("PODMAN_OPTS", "") end if not os.getenv("PODMAN_COMMAND_OPTS") then setenv ("PODMAN_COMMAND_OPTS", "") end --- directory containing this modulefile (dynamically defined) -local moduleDir = myFileName():match("(.*[/])") or "." +-- directory containing this modulefile, once symlinks resolved (dynamically defined) +local moduleDir = subprocess("realpath " .. myFileName()):match("(.*[/])") or "." -- interactive shell to any container, plus exec for aliases local containerPath = '{{ image }}' diff --git a/shpc/main/modules/templates/docker.tcl b/shpc/main/modules/templates/docker.tcl index ade20ceb4..93a25781c 100644 --- a/shpc/main/modules/templates/docker.tcl +++ b/shpc/main/modules/templates/docker.tcl @@ -55,8 +55,8 @@ set helpcommand "This module is a {{ docker }} container wrapper for {{ name }} {% if labels %}{% for key, value in labels.items() %}set {{ key }} "{{ value }}" {% endfor %}{% endif %} -# directory containing this modulefile (dynamically defined) -set moduleDir "[file dirname ${ModulesCurrentModulefile}]" +# directory containing this modulefile, once symlinks resolved (dynamically defined) +set moduleDir [file dirname [expr { [string equal [file type ${ModulesCurrentModulefile}] "link"] ? [file readlink ${ModulesCurrentModulefile}] : ${ModulesCurrentModulefile} }]] # conflict with modules with the same alias name conflict {{ parsed_name.tool }} diff --git a/shpc/main/modules/templates/includes/default_version.lua b/shpc/main/modules/templates/includes/default_version.lua new file mode 100644 index 000000000..a680e15c6 --- /dev/null +++ b/shpc/main/modules/templates/includes/default_version.lua @@ -0,0 +1,5 @@ +{% if not settings.default_version %}if (mode() == "load") then + if ( myModuleUsrName() ~= myModuleFullName() and myModuleUsrName() ~= string.gsub(myModuleFullName(),"/module$","") ) then + LmodError("Default modules are disabled by your systems administrator. You must specify `module load /`.") + end +end{% endif %} diff --git a/shpc/main/modules/templates/singularity.lua b/shpc/main/modules/templates/singularity.lua index f50c7ec4f..b7d3dc960 100644 --- a/shpc/main/modules/templates/singularity.lua +++ b/shpc/main/modules/templates/singularity.lua @@ -36,10 +36,11 @@ For each of the above, you can export: - SINGULARITY_COMMAND_OPTS: to define custom options for the command (e.g., -b) ]]) +{% include "includes/default_version.lua" %} {% if settings.singularity_module %}load("{{ settings.singularity_module }}"){% endif %} --- directory containing this modulefile (dynamically defined) -local moduleDir = myFileName():match("(.*[/])") or "." +-- directory containing this modulefile, once symlinks resolved (dynamically defined) +local moduleDir = subprocess("realpath " .. myFileName()):match("(.*[/])") or "." -- singularity environment variable to set shell setenv("SINGULARITY_SHELL", "{{ settings.singularity_shell }}") diff --git a/shpc/main/modules/templates/singularity.tcl b/shpc/main/modules/templates/singularity.tcl index dad790d99..b2e078dd5 100644 --- a/shpc/main/modules/templates/singularity.tcl +++ b/shpc/main/modules/templates/singularity.tcl @@ -59,8 +59,8 @@ set helpcommand "This module is a singularity container wrapper for {{ name }} v {% if labels %}{% for key, value in labels.items() %}set {{ key }} "{{ value }}" {% endfor %}{% endif %} -# directory containing this modulefile (dynamically defined) -set moduleDir "[file dirname ${ModulesCurrentModulefile}]" +# directory containing this modulefile, once symlinks resolved (dynamically defined) +set moduleDir [file dirname [expr { [string equal [file type ${ModulesCurrentModulefile}] "link"] ? [file readlink ${ModulesCurrentModulefile}] : ${ModulesCurrentModulefile} }]] # conflict with modules with the same alias name conflict {{ parsed_name.tool }} diff --git a/shpc/main/schemas.py b/shpc/main/schemas.py index 9294c3cca..6d5de0f81 100644 --- a/shpc/main/schemas.py +++ b/shpc/main/schemas.py @@ -139,6 +139,8 @@ ] }, "enable_tty": {"type": "boolean"}, + "symlink_base": {"type": ["string", "null"]}, + "symlink_tree": {"type": "boolean"}, "wrapper_scripts": wrapper_scripts, "container_tech": {"type": "string", "enum": ["singularity", "podman", "docker"]}, "singularity_shell": {"type": "string", "enum": shells}, diff --git a/shpc/main/settings.py b/shpc/main/settings.py index 73cf3cadf..454b6e7e0 100644 --- a/shpc/main/settings.py +++ b/shpc/main/settings.py @@ -19,6 +19,16 @@ from datetime import datetime import jsonschema import os +import re + +# Legacy values we still support for backwards compatibility, +# and a mapping to the new value that should be used. +legacy_values = { + "default_version": { + False: None, + True: "module_sys", + }, +} def OrderedList(*l): @@ -116,6 +126,13 @@ def load(self, settings_file=None): with open(self.settings_file, "r") as fd: self._settings.update(yaml.load(fd.read())) + # Upgrade legacy values to their new counterpart + for key in legacy_values: + current_value = self.get(key) + if current_value in legacy_values[key]: + new_value = legacy_values[key][current_value] + self.set(key, new_value) + def get(self, key, default=None): """ Get a settings value, doing appropriate substitution and expansion. @@ -276,6 +293,49 @@ def __iter__(self): for key, value in self.__dict__.items(): yield key, value + def update_params(self, params): + """ + Update a configuration on the fly (no save) only for set/add/remove. + Unlike the traditional set/get/add functions, this function expects + each entry in the params list to start with the action, e.g.: + + set:name:value + add:name:value + rm:name:value + """ + # Cut out early if params not provided + if not params: + return + + for param in params: + if not re.search("^(add|set|rm)", param, re.IGNORECASE) or ":" not in param: + logger.warning( + "Parameter update request must start with (add|set|rm):, skipping %s" + ) + command, param = param.split(":", 1) + self.update_param(command.lower(), param) + + def update_param(self, command, param): + """ + Given a parameter, update the configuration on the fly if it's in set/add/remove + """ + if ":" not in param: + logger.warning( + "Param %s is missing a :, should be key:value pair. Skipping." % param + ) + return + + key, value = param.split(":", 1) + if command == "set": + self.set(key, value) + logger.info("Updated %s to be %s" % (key, value)) + elif command == "add": + self.add(key, value) + logger.info("Added %s to %s" % (key, value)) + elif command == "remove": + self.remove(key, value) + logger.info("Removed %s from %s" % (key, value)) + class Settings(SettingsBase): """ diff --git a/shpc/settings.yml b/shpc/settings.yml index 68af84065..c0b3c4b07 100644 --- a/shpc/settings.yml +++ b/shpc/settings.yml @@ -38,6 +38,13 @@ default_version: module_sys # It's recommended to do this for faster loading container_base: $root_dir/containers +# If defined, create a simplified "symlink install" that shortens module names +# to be just the container, e.g., ghcr.io/autamus/samtools -> module load samtools +symlink_base: $root_dir/symlinks + +# Always generate a symlink, even without the command line argument +symlink_tree: false + # if defined, add to lmod script to load this Singularity module first singularity_module: podman_module: diff --git a/shpc/utils/__init__.py b/shpc/utils/__init__.py index 383f267f2..12daab62c 100644 --- a/shpc/utils/__init__.py +++ b/shpc/utils/__init__.py @@ -15,7 +15,7 @@ get_tmpfile, mkdir_p, mkdirp, - rmdir_to_base, + remove_to_base, print_json, read_file, read_json, diff --git a/shpc/utils/fileio.py b/shpc/utils/fileio.py index a1b8fa167..19e2303a5 100644 --- a/shpc/utils/fileio.py +++ b/shpc/utils/fileio.py @@ -61,7 +61,7 @@ def mkdir_p(path): logger.exit("Error creating path %s, exiting." % path) -def rmdir_to_base(path, base_path): +def remove_to_base(path, base_path): """ Delete the tree under $path and all the parents up to $base_path as long as they are empty @@ -71,7 +71,9 @@ def rmdir_to_base(path, base_path): if not path.startswith(base_path): logger.exit("Error: %s is not a parent of %s" % (base_path, path)) - if os.path.exists(path): + if os.path.islink(path) or os.path.isfile(path): + os.unlink(path) + elif os.path.isdir(path): shutil.rmtree(path) # If directories above it are empty, remove diff --git a/shpc/utils/terminal.py b/shpc/utils/terminal.py index 1fad12226..b309f46d2 100644 --- a/shpc/utils/terminal.py +++ b/shpc/utils/terminal.py @@ -95,7 +95,7 @@ def confirm_action(question, force=False): if force is True: return True - response = input(question) + response = input(question + " (yes/no)?") while len(response) < 1 or response[0].lower().strip() not in "ynyesno": response = input("Please answer yes or no: ") diff --git a/shpc/version.py b/shpc/version.py index 45f821506..0a62776fd 100644 --- a/shpc/version.py +++ b/shpc/version.py @@ -2,7 +2,7 @@ __copyright__ = "Copyright 2021-2022, Vanessa Sochat" __license__ = "MPL 2.0" -__version__ = "0.0.51" +__version__ = "0.0.52" AUTHOR = "Vanessa Sochat" NAME = "singularity-hpc" PACKAGE_URL = "https://github.com/singularityhub/singularity-hpc"