Tools install without fabric for bcbio

chapmanb · chapmanb · commit 5fa9af75851e · 2019-02-02T05:01:56.000-05:00
Move away from fabric requirement as a first step towards
supporting python 3 installs. Fabric 2.x has py3 support but requires a
full migration of fabric support. Since we're no longer making use
of cross host capabilities, this transitions to using subprocess
and local calls.

This is step 1, which provides a target for tool installation
without needing fabric.
diff --git a/cloudbio/manifest.py b/cloudbio/manifest.py
@@ -66,12 +66,20 @@ def write_custom_pkg_info(out_dir, tooldir):
     if not os.path.exists(out_file):
         out = {}
         for modname in custom_names:
-            mod = getattr(__import__("cloudbio.custom", globals(), locals(),
-                                     [modname], -1),
-                          modname)
-            for prog in [x for x in dir(mod) if x.startswith("install")]:
-                pkg = _get_custom_pkg_info(prog, getattr(mod, prog))
-                out[pkg["name"]] = pkg
+            try:
+                mod = getattr(__import__("cloudbio.custom", globals(), locals(),
+                                         [modname], -1),
+                              modname)
+            except ImportError as msg:
+                # Skip fabric import errors as we transition away from it
+                if "fabric" in str(msg):
+                    mod = None
+                else:
+                    raise
+            if mod:
+                for prog in [x for x in dir(mod) if x.startswith("install")]:
+                    pkg = _get_custom_pkg_info(prog, getattr(mod, prog))
+                    out[pkg["name"]] = pkg
         with open(out_file, "w") as out_handle:
             yaml.safe_dump(out, out_handle, default_flow_style=False, allow_unicode=False)
     return out_file
diff --git a/cloudbio/package/__init__.py b/cloudbio/package/__init__.py
@@ -2,20 +2,19 @@
 """
 import os
 
-from fabric.api import env, cd
-
-from cloudbio.custom.shared import _make_tmp_dir
-from cloudbio.package import brew
-from cloudbio.package.deb import (_apt_packages, _add_apt_gpg_keys,
-                                  _setup_apt_automation, _setup_apt_sources)
-from cloudbio.package.rpm import (_yum_packages, _setup_yum_bashrc,
-                                  _setup_yum_sources)
 
 def _configure_and_install_native_packages(env, pkg_install):
     """
     Setups up native package repositories, determines list
     of native packages to install, and installs them.
     """
+    from fabric.api import env
+    from cloudbio.package import brew
+    from cloudbio.package.deb import (_apt_packages, _add_apt_gpg_keys,
+                                      _setup_apt_automation, _setup_apt_sources)
+    from cloudbio.package.rpm import (_yum_packages, _setup_yum_bashrc,
+                                      _setup_yum_sources)
+
     home_dir = env.safe_run("echo $HOME")
     if home_dir:
         if env.shell_config.startswith("~"):
@@ -43,6 +42,7 @@ def _connect_native_packages(env, pkg_install, lib_install):
     This helps setup a non-sudo environment to handle software
     that needs a local version in our non-root directory tree.
     """
+    from fabric.api import env
     bin_dir = os.path.join(env.system_install, "bin")
     exports = _get_shell_exports(env)
     path = env.safe_run_output("echo $PATH")
@@ -70,6 +70,9 @@ def _print_shell_exports(env):
 def _create_local_virtualenv(target_dir):
     """Create virtualenv in target directory for non-sudo installs.
     """
+    from fabric.api import cd
+    from cloudbio.custom.shared import _make_tmp_dir
+
     url = "https://raw.github.com/pypa/virtualenv/master/virtualenv.py"
     if not os.path.exists(os.path.join(target_dir, "bin", "python")):
         with _make_tmp_dir() as work_dir:
diff --git a/cloudbio/package/conda.py b/cloudbio/package/conda.py
@@ -7,97 +7,105 @@
 import subprocess
 import yaml
 
-from distutils.version import LooseVersion
-
-from cloudbio.custom import shared
-from cloudbio.fabutils import quiet
-from cloudbio.flavor.config import get_config_file
 from cloudbio.package.shared import _yaml_to_packages
 
 ENV_PY_VERSIONS = collections.defaultdict(lambda: "python=2")
 ENV_PY_VERSIONS["python3"] = "python=3"
 
 def install_packages(env, to_install=None, packages=None):
+    """Old installation, based on pre-configured fabric inputs.
+    """
+    from cloudbio.flavor.config import get_config_file
+    from cloudbio.custom import shared
+
     if shared._is_anaconda(env):
         conda_bin = shared._conda_cmd(env)
         if hasattr(env, "conda_yaml"):
             Config = collections.namedtuple("Config", "base dist")
             config_file = Config(base=env.conda_yaml, dist=None)
         else:
             config_file = get_config_file(env, "packages-conda.yaml")
-        if config_file.base is None and packages is None:
-            packages = []
-            channels = ""
-        else:
-            if to_install:
-                (packages, _) = _yaml_to_packages(config_file.base, to_install, config_file.dist)
-            with open(config_file.base) as in_handle:
-                channels = " ".join(["-c %s" % x for x in yaml.safe_load(in_handle).get("channels", [])])
-        conda_envs = _create_environments(env, conda_bin, packages)
-        for env_dir in conda_envs.values():
-            _clean_environment(env_dir)
-        conda_info = json.loads(env.safe_run_output("{conda_bin} info --json".format(**locals())))
-        # Uninstall old R packages that clash with updated versions
-        # Temporary fix to allow upgrades from older versions that have migrated
-        # r-tximport is now bioconductor-tximport
-        # py2cairo is incompatible with r 3.4.1
-        for problem in ["r-tximport", "py2cairo", "libedit"]:
-            cur_packages = [x["name"] for x in
-                            json.loads(env.safe_run_output("{conda_bin} list --json {problem}".format(**locals())))]
-            if problem in cur_packages:
-                env.safe_run("{conda_bin} remove --force -y {problem}".format(**locals()))
-        # install our customized packages
-        if len(packages) > 0:
-            for env_name, env_packages in _split_by_condaenv(packages):
-                if env_name:
-                    assert env_name in conda_envs, (env_name, conda_envs)
-                    env_str = "-n %s" % env_name
-                else:
-                    env_str = ""
-                pkgs_str = " ".join(["'%s'" % x for x in sorted(env_packages)])
-                py_version = ENV_PY_VERSIONS[env_name]
-                if "deepvariant" in env_packages:
-                    # Ignore /etc/boto.cfg which creates conflicts with conda gsutils
-                    # https://github.com/GoogleCloudPlatform/gsutil/issues/516
-                    exports = "export BOTO_CONFIG=/ignoreglobal && "
-                else:
-                    exports = ""
-                env.safe_run("{exports}{conda_bin} install -y {env_str} {channels} "
-                             "{py_version} {pkgs_str}".format(**locals()))
-                conda_pkg_list = json.loads(env.safe_run_output(
-                    "{conda_bin} list --json {env_str}".format(**locals())))
-                for package in env_packages:
-                    _link_bin(package, env, conda_info, conda_bin, conda_pkg_list,
-                              conda_envdir=conda_envs.get(env_name))
-        conda_pkg_list = json.loads(env.safe_run_output("{conda_bin} list --json".format(**locals())))
-        for pkg in ["python", "conda", "pip"]:
-            _link_bin(pkg, env, conda_info, conda_bin, conda_pkg_list, files=[pkg], prefix="bcbio_")
-
-def _link_bin(package, env, conda_info, conda_bin, conda_pkg_list, files=None, prefix="", conda_env=None,
-              conda_envdir=None):
+        install_in(conda_bin, env.system_install, config_file.base, packages)
+
+def install_in(conda_bin, system_installdir, config_file=None, packages=None):
+    """Install packages inside a given anaconda directory.
+
+    New approach, local only and not dependent on fabric.
+    """
+    if config_file is None and packages is None:
+        packages = []
+        channels = ""
+    else:
+        (packages, _) = _yaml_to_packages(config_file)
+        with open(config_file) as in_handle:
+            channels = " ".join(["-c %s" % x for x in yaml.safe_load(in_handle).get("channels", [])])
+    conda_envs = _create_environments(conda_bin, packages)
+    for env_dir in conda_envs.values():
+        _clean_environment(env_dir)
+    conda_info = json.loads(subprocess.check_output("{conda_bin} info --json".format(**locals()), shell=True))
+    # Uninstall old R packages that clash with updated versions
+    # Temporary fix to allow upgrades from older versions that have migrated
+    # r-tximport is now bioconductor-tximport
+    # py2cairo is incompatible with r 3.4.1
+    problems = ["r-tximport", "py2cairo", "libedit"]
+    if problems:
+        print("Checking for problematic packages: %s" % ", ".join(problems))
+        cur_packages = [x["name"] for x in
+                        json.loads(subprocess.check_output("%s list --json '%s'" % (conda_bin, "|".join(problems)),
+                                                           shell=True)) if x["name"] in problems]
+        for problem in cur_packages:
+            subprocess.check_call("{conda_bin} remove --force -y {problem}".format(**locals()), shell=True)
+    # install our customized packages
+    if len(packages) > 0:
+        for env_name, env_packages in _split_by_condaenv(packages):
+            print("# Installing into conda environment %s: %s" % (env_name or "default", ", ".join(env_packages)))
+            if env_name:
+                assert env_name in conda_envs, (env_name, conda_envs)
+                env_str = "-n %s" % env_name
+            else:
+                env_str = ""
+            pkgs_str = " ".join(["'%s'" % x for x in sorted(env_packages)])
+            py_version = ENV_PY_VERSIONS[env_name]
+            if "deepvariant" in env_packages:
+                # Ignore /etc/boto.cfg which creates conflicts with conda gsutils
+                # https://github.com/GoogleCloudPlatform/gsutil/issues/516
+                exports = "export BOTO_CONFIG=/ignoreglobal && "
+            else:
+                exports = ""
+            subprocess.check_call("{exports}{conda_bin} install -y {env_str} {channels} "
+                                    "{py_version} {pkgs_str}".format(**locals()), shell=True)
+            conda_pkg_list = json.loads(subprocess.check_output(
+                "{conda_bin} list --json {env_str}".format(**locals()), shell=True))
+            for package in env_packages:
+                _link_bin(package, system_installdir, conda_info, conda_bin, conda_pkg_list,
+                            conda_envdir=conda_envs.get(env_name))
+    conda_pkg_list = json.loads(subprocess.check_output("{conda_bin} list --json".format(**locals()), shell=True))
+    for pkg in ["python", "conda", "pip"]:
+        _link_bin(pkg, system_installdir, conda_info, conda_bin, conda_pkg_list, files=[pkg], prefix="bcbio_")
+
+def _link_bin(package, system_installdir, conda_info, conda_bin, conda_pkg_list, files=None,
+              prefix="", conda_env=None, conda_envdir=None):
     """Link files installed in the bin directory into the install directory.
 
     This is imperfect but we're trying not to require injecting everything in the anaconda
     directory into a user's path.
     """
     package = package.split("=")[0]
-    final_bindir = os.path.join(env.system_install, "bin")
+    final_bindir = os.path.join(system_installdir, "bin")
     if conda_envdir:
         base_bindir = os.path.join(conda_envdir, "bin")
     else:
         base_bindir = os.path.dirname(conda_bin)
     # resolve any symlinks in the final and base heirarchies
-    with quiet():
-        final_bindir = env.safe_run_output("cd %s && pwd -P" % final_bindir)
-        base_bindir = env.safe_run_output("cd %s && pwd -P" % base_bindir)
+    final_bindir = subprocess.check_output("cd %s && pwd -P" % final_bindir, shell=True)
+    base_bindir = subprocess.check_output("cd %s && pwd -P" % base_bindir, shell=True)
     for pkg_subdir in [x for x in conda_pkg_list if x["name"] == package]:
         pkg_subdir = pkg_subdir["dist_name"].split("::")[-1]
         for pkg_dir in conda_info["pkgs_dirs"]:
             pkg_bindir = os.path.join(pkg_dir, pkg_subdir, "bin")
-            if env.safe_exists(pkg_bindir):
+            if os.path.exists(pkg_bindir):
                 if not files:
-                    with quiet():
-                        files = env.safe_run_output("ls -1 {pkg_bindir}".format(**locals())).split()
+                    files = subprocess.check_output("ls -1 {pkg_bindir}".format(**locals()), shell=True).split()
                 for fname in files:
                     # symlink to the original file in the /anaconda/bin directory
                     # this could be a hard or soft link
@@ -139,13 +147,13 @@ def _split_by_condaenv(packages):
             if k == "env":
                 condaenv = v
         out[condaenv].append(name)
-    return dict(out).items()
+    return sorted(dict(out).items())
 
-def _get_conda_envs(env, conda_bin):
-    info = json.loads(env.safe_run_output("{conda_bin} info --envs --json".format(**locals())))
+def _get_conda_envs(conda_bin):
+    info = json.loads(subprocess.check_output("{conda_bin} info --envs --json".format(**locals()), shell=True))
     return [e for e in info["envs"] if e.startswith(info["conda_prefix"])]
 
-def _create_environments(env, conda_bin, packages):
+def _create_environments(conda_bin, packages):
     """Creates custom local environments that conflict with global dependencies.
 
     Available environments:
@@ -157,17 +165,19 @@ def _create_environments(env, conda_bin, packages):
     """
     env_names = set([e for e, ps in _split_by_condaenv(packages) if e])
     out = {}
-    conda_envs = _get_conda_envs(env, conda_bin)
+    conda_envs = _get_conda_envs(conda_bin)
     if "python3" in env_names:
         if not any(x.endswith("/python3") for x in conda_envs):
-            env.safe_run("{conda_bin} create --no-default-packages -y --name python3 python=3".format(**locals()))
-            conda_envs = _get_conda_envs(env, conda_bin)
+            subprocess.check_call("{conda_bin} create --no-default-packages -y --name python3 python=3"
+                                  .format(**locals()), shell=True)
+            conda_envs = _get_conda_envs(conda_bin)
         out["python3"] = [x for x in conda_envs if x.endswith("/python3")][0]
     for addenv in ["samtools0", "dv"]:
         if addenv in env_names:
             if not any(x.endswith("/%s" % addenv) for x in conda_envs):
-                env.safe_run("{conda_bin} create --no-default-packages -y --name {addenv} python=2".format(**locals()))
-                conda_envs = _get_conda_envs(env, conda_bin)
+                subprocess.check_call("{conda_bin} create --no-default-packages -y --name {addenv} python=2"
+                                      .format(**locals()), shell=True)
+                conda_envs = _get_conda_envs(conda_bin)
             out[addenv] = [x for x in conda_envs if x.endswith("/%s" % addenv)][0]
     return out
 
diff --git a/cloudbio/package/shared.py b/cloudbio/package/shared.py
@@ -1,13 +1,11 @@
 """Shared functionality useful for multiple package managers.
 """
 import yaml
-from fabric.api import *
-from fabric.contrib.files import *
 
-def _yaml_to_packages(yaml_file, to_install=None, subs_yaml_file=None, namesort=True):
+def _yaml_to_packages(yaml_file, to_install=None, subs_yaml_file=None, namesort=True, env=None):
     """Read a list of packages from a nested YAML configuration file.
     """
-    env.logger.info("Reading %s" % yaml_file)
+    print("Reading packages from %s" % yaml_file)
     with open(yaml_file) as in_handle:
         full_data = yaml.load(in_handle)
         if full_data is None:
@@ -19,7 +17,7 @@ def _yaml_to_packages(yaml_file, to_install=None, subs_yaml_file=None, namesort=
         subs = {}
     # filter the data based on what we have configured to install
     data = [(k, v) for (k, v) in full_data.iteritems()
-            if to_install is None or k in to_install]
+            if (to_install is None or k in to_install) and k not in ["channels"]]
     data.sort()
     packages = []
     pkg_to_group = dict()
@@ -33,17 +31,16 @@ def _yaml_to_packages(yaml_file, to_install=None, subs_yaml_file=None, namesort=
             elif isinstance(cur_info, dict):
                 for key, val in cur_info.iteritems():
                     # if we are okay, propagate with the top level key
-                    if key == 'needs_64bit':
+                    if env and key == 'needs_64bit':
                         if env.is_64bit:
                             data.insert(0, (cur_key, val))
-                    elif key.startswith(env.distribution):
+                    elif env and key.startswith(env.distribution):
                         if key.endswith(env.dist_name):
                             data.insert(0, (cur_key, val))
                     else:
                         data.insert(0, (cur_key, val))
             else:
                 raise ValueError(cur_info)
-    env.logger.debug("Packages to install: {0}".format(",".join(packages)))
     return packages, pkg_to_group
 
 def _filter_subs_packages(initial, subs, namesort=True):
