Patches jra001k 20181227

.travis.yml

@@ -10,6 +10,7 @@ env:
   - TOX_ENV=qunit
   - TOX_ENV=py27-first_startup
   - TOX_ENV=py27-lint_docstring_include_list
+  - TOX_ENV=cwl_green
 
 matrix:
   include:

client/galaxy/scripts/mvc/form/form-parameters.js

@@ -4,6 +4,7 @@
 import { getGalaxyInstance } from "app";
 import Utils from "utils/utils";
 import Ui from "mvc/ui/ui-misc";
+import UiField from "mvc/ui/ui-field";
 import SelectContent from "mvc/ui/ui-select-content";
 import SelectLibrary from "mvc/ui/ui-select-library";
 import SelectFtp from "mvc/ui/ui-select-ftp";
@@ -34,6 +35,7 @@ export default Backbone.Model.extend({
         ftpfile: "_fieldFtp",
         upload: "_fieldUpload",
         rules: "_fieldRulesEdit",
+        field: "_fieldField",
         genomespacefile: "_fieldGenomeSpace"
     },
 
@@ -233,5 +235,13 @@ export default Backbone.Model.extend({
             id: `field-${input_def.id}`,
             onchange: input_def.onchange
         });
+    },
+
+    _fieldField: function(input_def) {
+        return new UiField({
+            id: `field-${input_def.id}`,
+            onchange: input_def.onchange
+        });
     }
+
 });

client/galaxy/scripts/mvc/ui/ui-field.js

@@ -0,0 +1,96 @@
+/** Renders the color picker used e.g. in the tool form **/
+import Utils from "utils/utils";
+import Ui from "mvc/ui/ui-misc";
+
+/** Renders an input element used e.g. in the tool form */
+export default Backbone.View.extend({
+    initialize: function(options) {
+        this.model =
+            (options && options.model) ||
+            new Backbone.Model({
+                value: {"src": "json", "value": null, "representation": "null"},
+            }).set(options);
+        this.$el = $("<div><p>moo cow</p></div>").addClass("ui-field");
+        console.log(this.model.get("value"));
+        var menuButton = new Ui.ButtonMenu({
+            id: "options",
+            icon: "fa-caret-down",
+            title: "Input Type",
+            tooltip: "View available input type options"
+        });
+        menuButton.addMenu({
+            title: "Integer",
+            onclick: () => {
+                this._changeType("integer")
+            }
+        });
+        menuButton.addMenu({
+            title: "Leave Unselected",
+            onclick: () => {
+                this._changeType("null")
+            }
+        });
+        this.$menuButton = menuButton;
+        this.$inputDiv = $("<div/>").addClass("select-input");
+
+        this.$el.append(menuButton.$el);
+        this.$el.append(this.$inputDiv);
+        this.setElement(this.$el);
+        this.listenTo(this.model, "change", this.render, this);
+        this.render();
+    },
+    value: function(new_val) {
+        var options = this.model.attributes;
+        if (new_val) {
+            this.model.set("value", new_val);
+            this.model.trigger("change");
+            options.onchange(new_val);
+        }
+        return this.model.get("value");
+    },
+    render: function() {
+        const value = this.model.get("value");
+        const rep = value.representation;
+        if ( rep == "null" ) {
+            this.$inputDiv.html($("<p>No value selected (null)</p>"));
+        } else if ( rep == "integer" ) {
+            const tagName = this.model.get("area") ? "textarea" : "input";
+            this.$inputDiv.html($(`<${tagName} value="${value.value}"/>`));
+            console.log(this.$inputDiv.find("input"));
+            this.$inputDiv.find("input").on("change", () => { this._onchange() });
+        }
+        return this;
+    },
+    _changeType: function(representation) {
+        const previousValue = this.model.get("value");
+        const previousRawValue = previousValue.value;
+        if ( representation == "null" ) {
+            this.model.set("value", {"src": "json", "value": null, "representation": "null"});
+        } else if ( representation == "integer" ) {
+            var value = parseInt(previousRawValue);
+            if ( isNaN( value ) ) {
+                value = 0;
+            }
+            this.model.set("value", {"src": "json", "value": 0, "representation": "integer"});
+        }
+    },
+    _rawValue: function(previousValue) {
+        const rep = previousValue.representation;
+        let rawVal;
+        if ( rep == "null" ) {
+            rawVal = null;
+        } else if ( rep == "integer" ) {
+            rawVal = parseInt(this.$inputDiv.find("input").val());
+        }
+        console.log("_rawValue returning " + rawVal);
+        return rawVal;
+
+    },
+    _onchange: function() {
+        const previousValue = this.model.get("value");
+        const newValue = this._rawValue(previousValue);
+        previousValue["value"] = newValue;
+        this.value(previousValue);
+        this.model.get("onchange") && this.model.get("onchange")(this.model.get("value"));
+    }
+});

config/datatypes_conf.xml.sample

@@ -202,7 +202,7 @@
     <datatype extension="tar" type="galaxy.datatypes.binary:CompressedArchive" subclass="true" display_in_upload="true">
       <converter file="tar_to_directory.xml" target_datatype="directory"/>
     </datatype>
-    <datatype extension="directory" type="galaxy.datatypes.data:Directory">
+    <datatype extension="directory" type="galaxy.datatypes.data:Directory" display_in_upload="true">
     </datatype>
     <!-- Proteomics Datatypes -->
     <datatype extension="pepxml" type="galaxy.datatypes.proteomics:PepXml" mimetype="application/xml" display_in_upload="true"/>
@@ -395,6 +395,7 @@
     <!-- End RGenetics Datatypes -->
     <datatype extension="ipynb" type="galaxy.datatypes.text:Ipynb" display_in_upload="true"/>
     <datatype extension="json" type="galaxy.datatypes.text:Json" display_in_upload="true"/>
+    <datatype extension="expression.json" type="galaxy.datatypes.text:ExpressionJson" display_in_upload="true"/>
     <!-- graph datatypes -->
     <datatype extension="xgmml" type="galaxy.datatypes.graph:Xgmml" display_in_upload="true"/>
     <datatype extension="sif" type="galaxy.datatypes.graph:Sif" display_in_upload="true"/>

config/tool_conf.xml.sample

@@ -32,9 +32,11 @@
     <tool file="${model_tools_path}/zip_collection.xml" />
     <tool file="${model_tools_path}/filter_failed_collection.xml" />
     <tool file="${model_tools_path}/filter_empty_collection.xml" />
+    <tool file="${model_tools_path}/filter_collection.xml" />
     <tool file="${model_tools_path}/flatten_collection.xml" />
     <tool file="${model_tools_path}/merge_collection.xml" />
     <tool file="${model_tools_path}/relabel_from_file.xml" />
+    <tool file="${model_tools_path}/group_collection.xml" />
     <tool file="${model_tools_path}/filter_from_file.xml" />
     <tool file="${model_tools_path}/sort_collection_list.xml" />
     <tool file="${model_tools_path}/tag_collection_from_file.xml" />

lib/galaxy/app.py

@@ -17,6 +17,7 @@
 from galaxy.managers.histories import HistoryManager
 from galaxy.managers.libraries import LibraryManager
 from galaxy.managers.tags import GalaxyTagManager
+from galaxy.managers.tools import DynamicToolManager
 from galaxy.openid.providers import OpenIDProviders
 from galaxy.queue_worker import GalaxyQueueWorker
 from galaxy.tools.cache import (
@@ -101,6 +102,7 @@ def __init__(self, **kwargs):
         self.test_data_resolver = test_data.TestDataResolver(file_dirs=self.config.tool_test_data_directories)
         self.library_folder_manager = FolderManager()
         self.library_manager = LibraryManager()
+        self.dynamic_tool_manager = DynamicToolManager(self)
 
         # Tool Data Tables
         self._configure_tool_data_tables(from_shed_config=False)

lib/galaxy/config.py

@@ -313,6 +313,7 @@ def __init__(self, **kwargs):
             log.warning("preserve_python_environment set to unknown value [%s], defaulting to legacy_only")
             preserve_python_environment = "legacy_only"
         self.preserve_python_environment = preserve_python_environment
+        self.nodejs_path = kwargs.get("nodejs_path", None)
         # Older default container cache path, I don't think anyone is using it anymore and it wasn't documented - we
         # should probably drop the backward compatiblity to save the path check.
         self.container_image_cache_path = self.resolve_path(kwargs.get("container_image_cache_path", "database/container_images"))
@@ -387,9 +388,10 @@ def __init__(self, **kwargs):
         # These are not even beta - just experiments - don't use them unless
         # you want yours tools to be broken in the future.
         self.enable_beta_tool_formats = string_as_bool(kwargs.get('enable_beta_tool_formats', 'False'))
+        # Should CWL artifacts be loaded with strict validation enabled.
+        self.strict_cwl_validation = string_as_bool(kwargs.get('strict_cwl_validation', 'True'))
         # Beta containers interface used by GIEs
         self.enable_beta_containers_interface = string_as_bool(kwargs.get('enable_beta_containers_interface', 'False'))
-
         # Certain modules such as the pause module will automatically cause
         # workflows to be scheduled in job handlers the way all workflows will
         # be someday - the following two properties can also be used to force this
@@ -1033,6 +1035,8 @@ def _configure_toolbox(self):
 
         self.citations_manager = CitationsManager(self)
 
+        from galaxy.managers.tools import DynamicToolManager
+        self.dynamic_tools_manager = DynamicToolManager(self)
         self._toolbox_lock = threading.RLock()
         # Initialize the tools, making sure the list of tool configs includes the reserved migrated_tools_conf.xml file.
         tool_configs = self.config.tool_configs

lib/galaxy/dataset_collections/builder.py

@@ -2,21 +2,23 @@
 from galaxy.util.odict import odict
 
 
-def build_collection(type, dataset_instances):
+def build_collection(type, dataset_instances, fields=None):
     """
     Build DatasetCollection with populated DatasetcollectionElement objects
     corresponding to the supplied dataset instances or throw exception if
     this is not a valid collection of the specified type.
     """
-    dataset_collection = model.DatasetCollection()
-    set_collection_elements(dataset_collection, type, dataset_instances)
+    dataset_collection = model.DatasetCollection(fields=fields)
+    set_collection_elements(dataset_collection, type, dataset_instances, fields=fields)
     return dataset_collection
 
 
-def set_collection_elements(dataset_collection, type, dataset_instances):
+def set_collection_elements(dataset_collection, type, dataset_instances, fields=None):
     element_index = 0
     elements = []
-    for element in type.generate_elements(dataset_instances):
+    if fields == "auto":
+        fields = guess_fields(dataset_instances)
+    for element in type.generate_elements(dataset_instances, fields=fields):
         element.element_index = element_index
         element.collection = dataset_collection
         elements.append(element)
@@ -28,6 +30,16 @@ def set_collection_elements(dataset_collection, type, dataset_instances):
     return dataset_collection
 
 
+def guess_fields(dataset_instances):
+    fields = []
+    for identifier, element in dataset_instances.items():
+        # TODO: Make generic enough to handle nested record types.
+        assert element.history_content_type == "dataset"
+        fields.append({"class": "File", "name": identifier})
+
+    return fields
+
+
 class CollectionBuilder(object):
     """ Purely functional builder pattern for building a dataset collection. """
 

lib/galaxy/dataset_collections/matching.py

@@ -15,8 +15,10 @@ class CollectionsToMatch(object):
 
     def __init__(self):
         self.collections = {}
+        self.uses_ephemeral_collections = False
 
     def add(self, input_name, hdca, subcollection_type=None, linked=True):
+        self.uses_ephemeral_collections = self.uses_ephemeral_collections or not hasattr(hdca, "hid")
         self.collections[input_name] = bunch.Bunch(
             hdca=hdca,
             subcollection_type=subcollection_type,
@@ -46,6 +48,7 @@ def __init__(self):
         self.collections = {}
         self.subcollection_types = {}
         self.action_tuples = {}
+        self.uses_ephemeral_collections = False
 
     def __attempt_add_to_linked_match(self, input_name, hdca, collection_type_description, subcollection_type):
         structure = get_structure(hdca, collection_type_description, leaf_subcollection_type=subcollection_type)
@@ -86,12 +89,21 @@ def map_over_action_tuples(self, input_name):
     def is_mapped_over(self, input_name):
         return input_name in self.collections
 
+    @property
+    def implicit_inputs(self):
+        if not self.uses_ephemeral_collections:
+            # Consider doing something smarter here.
+            return list(self.collections.items())
+        else:
+            return []
+
     @staticmethod
     def for_collections(collections_to_match, collection_type_descriptions):
         if not collections_to_match.has_collections():
             return None
 
         matching_collections = MatchingCollections()
+        matching_collections.uses_ephemeral_collections = collections_to_match.uses_ephemeral_collections
         for input_key, to_match in sorted(collections_to_match.items()):
             hdca = to_match.hdca
             collection_type_description = collection_type_descriptions.for_collection_type(hdca.collection.collection_type)

lib/galaxy/dataset_collections/registry.py

@@ -1,10 +1,15 @@
 from galaxy import model
 from .types import (
     list,
-    paired
+    paired,
+    record,
 )
 
-PLUGIN_CLASSES = [list.ListDatasetCollectionType, paired.PairedDatasetCollectionType]
+PLUGIN_CLASSES = [
+    list.ListDatasetCollectionType,
+    paired.PairedDatasetCollectionType,
+    record.RecordDatasetCollectionType,
+]
 
 
 class DatasetCollectionTypesRegistry(object):
@@ -15,12 +20,12 @@ def __init__(self, app):
     def get(self, plugin_type):
         return self.__plugins[plugin_type]
 
-    def prototype(self, plugin_type):
+    def prototype(self, plugin_type, fields=None):
         plugin_type_object = self.get(plugin_type)
         if not hasattr(plugin_type_object, 'prototype_elements'):
             raise Exception("Cannot pre-determine structure for collection of type %s" % plugin_type)
 
         dataset_collection = model.DatasetCollection()
-        elements = [e for e in plugin_type_object.prototype_elements()]
+        elements = [e for e in plugin_type_object.prototype_elements(fields=fields)]
         dataset_collection.elements = elements
         return dataset_collection

lib/galaxy/dataset_collections/type_description.py

@@ -7,9 +7,9 @@ def __init__(self, type_registry):
         # I think.
         self.type_registry = type_registry
 
-    def for_collection_type(self, collection_type):
+    def for_collection_type(self, collection_type, fields=None):
         assert collection_type is not None
-        return CollectionTypeDescription(collection_type, self)
+        return CollectionTypeDescription(collection_type, self, fields=fields)
 
 
 class CollectionTypeDescription(object):
@@ -43,9 +43,10 @@ class CollectionTypeDescription(object):
     'paired'
     """
 
-    def __init__(self, collection_type, collection_type_description_factory):
+    def __init__(self, collection_type, collection_type_description_factory, fields=None):
         self.collection_type = collection_type
         self.collection_type_description_factory = collection_type_description_factory
+        self.fields = fields
         self.__has_subcollections = self.collection_type.find(":") > 0
 
     def child_collection_type(self):
@@ -83,9 +84,13 @@ def has_subcollections_of_type(self, other_collection_type):
         collection_type = self.collection_type
         return collection_type.endswith(other_collection_type) and collection_type != other_collection_type
 
-    def is_subcollection_of_type(self, other_collection_type):
+    def is_subcollection_of_type(self, other_collection_type, proper=True):
+        """If proper is False, than a type is consider a subcollection of itself."""
         if not hasattr(other_collection_type, 'collection_type'):
             other_collection_type = self.collection_type_description_factory.for_collection_type(other_collection_type)
+        if not proper and self.can_match_type(other_collection_type):
+            return True
+
         return other_collection_type.has_subcollections_of_type(self)
 
     def can_match_type(self, other_collection_type):

lib/galaxy/dataset_collections/types/__init__.py

@@ -15,7 +15,7 @@
 class DatasetCollectionType(object):
 
     @abstractmethod
-    def generate_elements(self, dataset_instances):
+    def generate_elements(self, dataset_instances, **kwds):
         """ Generate DatasetCollectionElements with corresponding
         to the supplied dataset instances or throw exception if
         this is not a valid collection of the specified type.
@@ -24,5 +24,8 @@ def generate_elements(self, dataset_instances):
 
 class BaseDatasetCollectionType(DatasetCollectionType):
 
+    def __init__(self, **kwds):
+        pass
+
     def _validation_failed(self, message):
         raise exceptions.ObjectAttributeInvalidException(message)

lib/galaxy/dataset_collections/types/list.py

@@ -7,10 +7,7 @@ class ListDatasetCollectionType(BaseDatasetCollectionType):
     """
     collection_type = "list"
 
-    def __init__(self):
-        pass
-
-    def generate_elements(self, elements):
+    def generate_elements(self, elements, **kwds):
         for identifier, element in elements.items():
             association = DatasetCollectionElement(
                 element=element,