From 7c7d4627b3144ac78ed8e5ca9d23cee445442dd0 Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Wed, 6 Feb 2019 14:31:04 -0500 Subject: [PATCH 01/28] updated nipype2boutiques to conform to current schema, added default value for tool-version when interface version is null --- nipype/utils/nipype2boutiques.py | 36 ++++++++++++--------- nipype/utils/tests/test_nipype2boutiques.py | 5 +-- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/nipype/utils/nipype2boutiques.py b/nipype/utils/nipype2boutiques.py index 21ecbc0eee..617a5d28d2 100644 --- a/nipype/utils/nipype2boutiques.py +++ b/nipype/utils/nipype2boutiques.py @@ -25,15 +25,18 @@ def generate_boutiques_descriptor( - module, interface_name, ignored_template_inputs, docker_image, - docker_index, verbose, ignore_template_numbers): + module, interface_name, ignored_template_inputs, container_image, + container_index, container_type, verbose, ignore_template_numbers): ''' Returns a JSON string containing a JSON Boutiques description of a Nipype interface. Arguments: * module: module where the Nipype interface is declared. - * interface: Nipype interface. + * interface_name: name of Nipype interface. * ignored_template_inputs: a list of input names that should be ignored in the generation of output path templates. * ignore_template_numbers: True if numbers must be ignored in output path creations. + * container_image: name of the container image where the tool is installed + * container_index: optional index where the image is available + * container_type: type of container image (Docker or Singularity) ''' if not module: @@ -59,13 +62,15 @@ def generate_boutiques_descriptor( tool_desc[ 'description'] = interface_name + ", as implemented in Nipype (module: " + module_name + ", interface: " + interface_name + ")." tool_desc['inputs'] = [] - tool_desc['outputs'] = [] - tool_desc['tool-version'] = interface.version - tool_desc['schema-version'] = '0.2-snapshot' - if docker_image: - tool_desc['docker-image'] = docker_image - if docker_index: - tool_desc['docker-index'] = docker_index + tool_desc['output-files'] = [] + tool_desc['tool-version'] = interface.version if interface.version is not None else 'undefined' + tool_desc['schema-version'] = '0.5' + if container_image: + tool_desc['container-image'] = {} + tool_desc['container-image']['image'] = container_image + tool_desc['container-image']['type'] = container_type + if container_index: + tool_desc['container-image']['index'] = container_index # Generates tool inputs for name, spec in sorted(interface.inputs.traits(transient=None).items()): @@ -73,7 +78,7 @@ def generate_boutiques_descriptor( ignored_template_inputs, verbose, ignore_template_numbers) tool_desc['inputs'].append(input) - tool_desc['command-line'] += input['command-line-key'] + " " + tool_desc['command-line'] += input['value-key'] + " " if verbose: print("-> Adding input " + input['name']) @@ -82,13 +87,13 @@ def generate_boutiques_descriptor( output = get_boutiques_output(name, interface, tool_desc['inputs'], verbose) if output['path-template'] != "": - tool_desc['outputs'].append(output) + tool_desc['output-files'].append(output) if verbose: print("-> Adding output " + output['name']) elif verbose: print("xx Skipping output " + output['name'] + " with no path template.") - if tool_desc['outputs'] == []: + if tool_desc['output-files'] == []: raise Exception("Tool has no output.") # Removes all temporary values from inputs (otherwise they will @@ -125,7 +130,7 @@ def get_boutiques_input(inputs, interface, input_name, spec, input['name'] = input_name.replace('_', ' ').capitalize() input['type'] = get_type_from_spec_info(spec_info) input['list'] = is_list(spec_info) - input['command-line-key'] = "[" + input_name.upper( + input['value-key'] = "[" + input_name.upper( ) + "]" # assumes that input names are unique input['command-line-flag'] = ("--%s" % input_name + " ").strip() input['tempvalue'] = None @@ -178,7 +183,6 @@ def get_boutiques_output(name, interface, tool_inputs, verbose=False): output = {} output['name'] = name.replace('_', ' ').capitalize() output['id'] = name - output['type'] = "File" output['path-template'] = "" output[ 'optional'] = True # no real way to determine if an output is always produced, regardless of the input values. @@ -201,7 +205,7 @@ def get_boutiques_output(name, interface, tool_inputs, verbose=False): if str(input_value) in output_value: output_value = os.path.basename( output_value.replace(input_value, - input['command-line-key']) + input['value-key']) ) # FIXME: this only works if output is written in the current directory output['path-template'] = os.path.basename(output_value) return output diff --git a/nipype/utils/tests/test_nipype2boutiques.py b/nipype/utils/tests/test_nipype2boutiques.py index f1d0c46eed..3aecefda63 100644 --- a/nipype/utils/tests/test_nipype2boutiques.py +++ b/nipype/utils/tests/test_nipype2boutiques.py @@ -11,7 +11,8 @@ def test_generate(): generate_boutiques_descriptor(module='nipype.interfaces.ants.registration', interface_name='ANTS', ignored_template_inputs=(), - docker_image=None, - docker_index=None, + container_image=None, + container_index=None, + container_type=None, verbose=False, ignore_template_numbers=False) From fc27d01227127f1c61033e72c4922afdd1b03368 Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Wed, 6 Feb 2019 15:56:10 -0500 Subject: [PATCH 02/28] added a step to make sure output IDs are unique in case output name is the same as an input name --- nipype/utils/nipype2boutiques.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/nipype/utils/nipype2boutiques.py b/nipype/utils/nipype2boutiques.py index 617a5d28d2..626cf13067 100644 --- a/nipype/utils/nipype2boutiques.py +++ b/nipype/utils/nipype2boutiques.py @@ -182,7 +182,16 @@ def get_boutiques_output(name, interface, tool_inputs, verbose=False): """ output = {} output['name'] = name.replace('_', ' ').capitalize() - output['id'] = name + + # Check if the output name was already used as an input name + # If so, append '_outfile' to the end of the ID + unique_id = True + for inp in tool_inputs: + if inp['id'] == name: + unique_id = False + break + output['id'] = name if unique_id else name + '_outfile' + output['path-template'] = "" output[ 'optional'] = True # no real way to determine if an output is always produced, regardless of the input values. From e7739e74c1fc78efa6603fa09b510069b33be544 Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Thu, 7 Feb 2019 16:55:36 -0500 Subject: [PATCH 03/28] descriptor gets saved to a file for easier exporting, added value-choices extraction, added method to get input type from handler type --- nipype/utils/nipype2boutiques.py | 39 +++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/nipype/utils/nipype2boutiques.py b/nipype/utils/nipype2boutiques.py index 626cf13067..33eaeb56c1 100644 --- a/nipype/utils/nipype2boutiques.py +++ b/nipype/utils/nipype2boutiques.py @@ -101,6 +101,10 @@ def generate_boutiques_descriptor( for input in tool_desc['inputs']: del input['tempvalue'] + # Save descriptor to a file + with open(interface_name + '.json', 'w') as outfile: + json.dump(tool_desc, outfile) + return json.dumps(tool_desc, indent=4, separators=(',', ': ')) @@ -128,7 +132,10 @@ def get_boutiques_input(inputs, interface, input_name, spec, input = {} input['id'] = input_name input['name'] = input_name.replace('_', ' ').capitalize() - input['type'] = get_type_from_spec_info(spec_info) + + # Figure out the input type from its handler type + input['type'] = get_type_from_handler_type(spec.handler) + input['list'] = is_list(spec_info) input['value-key'] = "[" + input_name.upper( ) + "]" # assumes that input names are unique @@ -145,6 +152,14 @@ def get_boutiques_input(inputs, interface, input_name, spec, if spec.usedefault: input['default-value'] = spec.default_value()[1] + try: + value_choices = spec.handler.values + except AttributeError: + pass + else: + if value_choices is not None: + input['value-choices'] = value_choices + # Create unique, temporary value. temp_value = must_generate_value(input_name, input['type'], ignored_template_inputs, spec_info, spec, @@ -158,9 +173,9 @@ def get_boutiques_input(inputs, interface, input_name, spec, str(tempvalue)) # Now that temp values have been generated, set Boolean types to - # Number (there is no Boolean type in Boutiques) + # Flag (there is no Boolean type in Boutiques) if input['type'] == "Boolean": - input['type'] = "Number" + input['type'] = "Flag" return input @@ -217,9 +232,16 @@ def get_boutiques_output(name, interface, tool_inputs, verbose=False): input['value-key']) ) # FIXME: this only works if output is written in the current directory output['path-template'] = os.path.basename(output_value) + + if not output_value: + # Look for an input with the same name and use this as the path template + for input in tool_inputs: + if input['id'] == name: + output['path-template'] = input['value-key'] return output +# TODO remove this once we know get_type_from_handler_type works well def get_type_from_spec_info(spec_info): ''' Returns an input type from the spec info. There must be a better @@ -235,6 +257,17 @@ def get_type_from_spec_info(spec_info): return "String" +def get_type_from_handler_type(handler): + handler_type = type(handler).__name__ + if handler_type == "File" or handler_type == "Directory": + return "File" + elif handler_type == "Int" or handler_type == "Float": + return "Number" + elif handler_type == "Bool": + return "Flag" + else: + return "String" + def is_list(spec_info): ''' Returns True if the spec info looks like it describes a list From 181c481e003410e7e3f95156eaf04a1a0668c084 Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Thu, 7 Feb 2019 18:16:59 -0500 Subject: [PATCH 04/28] added warning message when cannot determine path template for output, added check for integer types, added method to get description from spec --- nipype/utils/nipype2boutiques.py | 66 +++++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 14 deletions(-) diff --git a/nipype/utils/nipype2boutiques.py b/nipype/utils/nipype2boutiques.py index 33eaeb56c1..83148fc10f 100644 --- a/nipype/utils/nipype2boutiques.py +++ b/nipype/utils/nipype2boutiques.py @@ -63,7 +63,7 @@ def generate_boutiques_descriptor( 'description'] = interface_name + ", as implemented in Nipype (module: " + module_name + ", interface: " + interface_name + ")." tool_desc['inputs'] = [] tool_desc['output-files'] = [] - tool_desc['tool-version'] = interface.version if interface.version is not None else 'undefined' + tool_desc['tool-version'] = interface.version if interface.version is not None else "No version provided." tool_desc['schema-version'] = '0.5' if container_image: tool_desc['container-image'] = {} @@ -84,7 +84,7 @@ def generate_boutiques_descriptor( # Generates tool outputs for name, spec in sorted(outputs.traits(transient=None).items()): - output = get_boutiques_output(name, interface, tool_desc['inputs'], + output = get_boutiques_output(outputs, name, spec, interface, tool_desc['inputs'], verbose) if output['path-template'] != "": tool_desc['output-files'].append(output) @@ -105,6 +105,8 @@ def generate_boutiques_descriptor( with open(interface_name + '.json', 'w') as outfile: json.dump(tool_desc, outfile) + print("NOTE: Descriptors produced by this script may not entirely conform to the Nipype interface " + "specs. Please check that the descriptor is correct before using it.") return json.dumps(tool_desc, indent=4, separators=(',', ': ')) @@ -125,8 +127,6 @@ def get_boutiques_input(inputs, interface, input_name, spec, Assumes that: * Input names are unique. """ - if not spec.desc: - spec.desc = "No description provided." spec_info = spec.full_info(inputs, input_name, None) input = {} @@ -134,17 +134,17 @@ def get_boutiques_input(inputs, interface, input_name, spec, input['name'] = input_name.replace('_', ' ').capitalize() # Figure out the input type from its handler type - input['type'] = get_type_from_handler_type(spec.handler) + input_type = get_type_from_handler_type(spec.handler) + input['type'] = input_type[0] + if input_type[1]: + input['integer'] = True input['list'] = is_list(spec_info) input['value-key'] = "[" + input_name.upper( ) + "]" # assumes that input names are unique input['command-line-flag'] = ("--%s" % input_name + " ").strip() input['tempvalue'] = None - input['description'] = spec_info.capitalize( - ) + ". " + spec.desc.capitalize() - if not input['description'].endswith('.'): - input['description'] += '.' + input['description'] = get_description_from_spec(inputs, input_name, spec) if not (hasattr(spec, "mandatory") and spec.mandatory): input['optional'] = True else: @@ -180,12 +180,14 @@ def get_boutiques_input(inputs, interface, input_name, spec, return input -def get_boutiques_output(name, interface, tool_inputs, verbose=False): +def get_boutiques_output(outputs, name, spec, interface, tool_inputs, verbose=False): """ Returns a dictionary containing the Boutiques output corresponding to a Nipype output. Args: + * outputs: outputs of the Nipype interface. * name: name of the Nipype output. + * spec: Nipype output spec. * interface: Nipype interface. * tool_inputs: list of tool inputs (as produced by method get_boutiques_input). @@ -211,9 +213,13 @@ def get_boutiques_output(name, interface, tool_inputs, verbose=False): output[ 'optional'] = True # no real way to determine if an output is always produced, regardless of the input values. + output['description'] = get_description_from_spec(outputs, name, spec) + # Path template creation. output_value = interface._list_outputs()[name] + + # If output value is defined, use its basename if output_value != "" and isinstance( output_value, str): # FIXME: this crashes when there are multiple output values. @@ -233,11 +239,20 @@ def get_boutiques_output(name, interface, tool_inputs, verbose=False): ) # FIXME: this only works if output is written in the current directory output['path-template'] = os.path.basename(output_value) + # If output value is undefined, create a placeholder for the path template if not output_value: # Look for an input with the same name and use this as the path template + found = False for input in tool_inputs: if input['id'] == name: output['path-template'] = input['value-key'] + found = True + break + # If no input with the same name was found, warn the user they should provide it manually + if not found: + print("WARNING: Could not determine path template for output %s. Please provide one for the " + "descriptor manually." % name) + output['path-template'] = "WARNING: No path template provided." return output @@ -258,15 +273,21 @@ def get_type_from_spec_info(spec_info): def get_type_from_handler_type(handler): + ''' + Gets the input type from the spec handler type. + Returns a tuple containing the type and a boolean to specify + if the type is an integer. + ''' handler_type = type(handler).__name__ + print("TYPE", handler_type) if handler_type == "File" or handler_type == "Directory": - return "File" + return "File", False elif handler_type == "Int" or handler_type == "Float": - return "Number" + return "Number", handler_type == "Int" elif handler_type == "Bool": - return "Flag" + return "Flag", False else: - return "String" + return "String", False def is_list(spec_info): ''' @@ -336,3 +357,20 @@ def must_generate_value(name, type, ignored_template_inputs, spec_info, spec, if not ignored_template_inputs: return True return not (name in ignored_template_inputs) + + +def get_description_from_spec(object, name, spec): + ''' + Generates a description based on the input or output spec. + ''' + if not spec.desc: + spec.desc = "No description provided." + spec_info = spec.full_info(object, name, None) + + boutiques_description = (spec_info.capitalize( + ) + ". " + spec.desc.capitalize()).replace("\n", '') + + if not boutiques_description.endswith('.'): + boutiques_description += '.' + + return boutiques_description From ce068151b7dcc0538f302557ac1b3af3cc86c03d Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Thu, 7 Feb 2019 19:50:29 -0500 Subject: [PATCH 05/28] fixed json file formatting, added save parameter, fixed command-line-flag to take argstr from input spec, added requires-inputs and disables-inputs --- nipype/utils/nipype2boutiques.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/nipype/utils/nipype2boutiques.py b/nipype/utils/nipype2boutiques.py index 83148fc10f..251269167b 100644 --- a/nipype/utils/nipype2boutiques.py +++ b/nipype/utils/nipype2boutiques.py @@ -26,7 +26,7 @@ def generate_boutiques_descriptor( module, interface_name, ignored_template_inputs, container_image, - container_index, container_type, verbose, ignore_template_numbers): + container_index, container_type, verbose, ignore_template_numbers, save): ''' Returns a JSON string containing a JSON Boutiques description of a Nipype interface. Arguments: @@ -37,6 +37,7 @@ def generate_boutiques_descriptor( * container_image: name of the container image where the tool is installed * container_index: optional index where the image is available * container_type: type of container image (Docker or Singularity) + * save: True if you want to save descriptor to a file ''' if not module: @@ -102,8 +103,11 @@ def generate_boutiques_descriptor( del input['tempvalue'] # Save descriptor to a file - with open(interface_name + '.json', 'w') as outfile: - json.dump(tool_desc, outfile) + if save: + with open(interface_name + '.json', 'w') as outfile: + json.dump(tool_desc, outfile, indent=4, separators=(',', ': ')) + if verbose: + print("-> Descriptor saved to file " + outfile.name) print("NOTE: Descriptors produced by this script may not entirely conform to the Nipype interface " "specs. Please check that the descriptor is correct before using it.") @@ -142,7 +146,14 @@ def get_boutiques_input(inputs, interface, input_name, spec, input['list'] = is_list(spec_info) input['value-key'] = "[" + input_name.upper( ) + "]" # assumes that input names are unique - input['command-line-flag'] = ("--%s" % input_name + " ").strip() + + # Add the command line flag specified by argstr + # If no argstr is provided and input type is Flag, create a flag from the name + if spec.argstr and spec.argstr.split("%")[0]: + input['command-line-flag'] = spec.argstr.split("%")[0].strip() + elif input['type'] == "Flag": + input['command-line-flag'] = ("--%s" % input_name + " ").strip() + input['tempvalue'] = None input['description'] = get_description_from_spec(inputs, input_name, spec) if not (hasattr(spec, "mandatory") and spec.mandatory): @@ -160,6 +171,12 @@ def get_boutiques_input(inputs, interface, input_name, spec, if value_choices is not None: input['value-choices'] = value_choices + if spec.requires is not None: + input['requires-inputs'] = spec.requires + + if spec.xor is not None: + input['disables-inputs'] = spec.xor + # Create unique, temporary value. temp_value = must_generate_value(input_name, input['type'], ignored_template_inputs, spec_info, spec, @@ -279,7 +296,6 @@ def get_type_from_handler_type(handler): if the type is an integer. ''' handler_type = type(handler).__name__ - print("TYPE", handler_type) if handler_type == "File" or handler_type == "Directory": return "File", False elif handler_type == "Int" or handler_type == "Float": From c1cfc80da1b85beafadb825251f840e700b7dbaa Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Thu, 7 Feb 2019 21:52:33 -0500 Subject: [PATCH 06/28] added logic to deal with range and list inputs, added try except block to deal with undefined output values --- nipype/utils/nipype2boutiques.py | 68 +++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 22 deletions(-) diff --git a/nipype/utils/nipype2boutiques.py b/nipype/utils/nipype2boutiques.py index 251269167b..f1a4f46017 100644 --- a/nipype/utils/nipype2boutiques.py +++ b/nipype/utils/nipype2boutiques.py @@ -138,12 +138,48 @@ def get_boutiques_input(inputs, interface, input_name, spec, input['name'] = input_name.replace('_', ' ').capitalize() # Figure out the input type from its handler type - input_type = get_type_from_handler_type(spec.handler) - input['type'] = input_type[0] - if input_type[1]: + handler_type = type(spec.handler).__name__ + + if handler_type == "File" or handler_type == "Directory": + input['type'] = "File" + elif handler_type == "Int": + input['type'] = "Number" input['integer'] = True + elif handler_type == "Float": + input['type'] = "Number" + elif handler_type == "Bool": + input['type'] = "Flag" + else: + input['type'] = "String" + + # Deal with range inputs + if handler_type == "Range": + input['type'] = "Number" + if spec.handler.low is not None: + input['minimum'] = spec.handler.low + if spec.handler.high is not None: + input['maximum'] = spec.handler.high + if spec.handler.exclude_low is not None: + input['exclusive-minimum'] = spec.handler.exclude_low + if spec.handler.exclude_high is not None: + input['exclusive-maximum'] = spec.handler.exclude_high + + # Deal with list inputs + if handler_type == "List": + input['list'] = True + trait_type = type(spec.handler.item_trait.trait_type).__name__ + if trait_type == "Int": + input['integer'] = True + input['type'] = "Number" + elif trait_type == "Float": + input['type'] = "Number" + else: + input['type'] = "String" + if spec.handler.minlen is not None: + input['min-list-entries'] = spec.handler.minlen + if spec.handler.maxlen is not None: + input['max-list-entries'] = spec.handler.maxlen - input['list'] = is_list(spec_info) input['value-key'] = "[" + input_name.upper( ) + "]" # assumes that input names are unique @@ -234,7 +270,10 @@ def get_boutiques_output(outputs, name, spec, interface, tool_inputs, verbose=Fa # Path template creation. - output_value = interface._list_outputs()[name] + try: + output_value = interface._list_outputs()[name] + except TypeError: + output_value = None # If output value is defined, use its basename if output_value != "" and isinstance( @@ -273,7 +312,7 @@ def get_boutiques_output(outputs, name, spec, interface, tool_inputs, verbose=Fa return output -# TODO remove this once we know get_type_from_handler_type works well +# TODO remove this def get_type_from_spec_info(spec_info): ''' Returns an input type from the spec info. There must be a better @@ -289,22 +328,7 @@ def get_type_from_spec_info(spec_info): return "String" -def get_type_from_handler_type(handler): - ''' - Gets the input type from the spec handler type. - Returns a tuple containing the type and a boolean to specify - if the type is an integer. - ''' - handler_type = type(handler).__name__ - if handler_type == "File" or handler_type == "Directory": - return "File", False - elif handler_type == "Int" or handler_type == "Float": - return "Number", handler_type == "Int" - elif handler_type == "Bool": - return "Flag", False - else: - return "String", False - +# TODO remove this def is_list(spec_info): ''' Returns True if the spec info looks like it describes a list From ee4b935b3b823df1b61254393632d9c3f7ce70c9 Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Mon, 11 Feb 2019 17:25:54 -0500 Subject: [PATCH 07/28] added logic to deal with compound traits, added check to make sure inputs don't disable themselves, made default output path template the output id --- nipype/utils/nipype2boutiques.py | 81 ++++++++++++++------- nipype/utils/tests/test_nipype2boutiques.py | 3 +- 2 files changed, 57 insertions(+), 27 deletions(-) diff --git a/nipype/utils/nipype2boutiques.py b/nipype/utils/nipype2boutiques.py index f1a4f46017..34b79437fa 100644 --- a/nipype/utils/nipype2boutiques.py +++ b/nipype/utils/nipype2boutiques.py @@ -78,10 +78,17 @@ def generate_boutiques_descriptor( input = get_boutiques_input(inputs, interface, name, spec, ignored_template_inputs, verbose, ignore_template_numbers) - tool_desc['inputs'].append(input) - tool_desc['command-line'] += input['value-key'] + " " - if verbose: - print("-> Adding input " + input['name']) + if isinstance(input, list): + for i in input: + tool_desc['inputs'].append(i) + tool_desc['command-line'] += i['value-key'] + " " + if verbose: + print("-> Adding input " + i['name']) + else: + tool_desc['inputs'].append(input) + tool_desc['command-line'] += input['value-key'] + " " + if verbose: + print("-> Adding input " + input['name']) # Generates tool outputs for name, spec in sorted(outputs.traits(transient=None).items()): @@ -116,7 +123,7 @@ def generate_boutiques_descriptor( def get_boutiques_input(inputs, interface, input_name, spec, ignored_template_inputs, verbose, - ignore_template_numbers): + ignore_template_numbers, handler=None, input_number=None): """ Returns a dictionary containing the Boutiques input corresponding to a Nipype intput. @@ -134,11 +141,32 @@ def get_boutiques_input(inputs, interface, input_name, spec, spec_info = spec.full_info(inputs, input_name, None) input = {} - input['id'] = input_name + + if input_number is not None: + input['id'] = input_name + "_" + str(input_number + 1) + else: + input['id'] = input_name + input['name'] = input_name.replace('_', ' ').capitalize() + if handler is None: + trait_handler = spec.handler + else: + trait_handler = handler + # Figure out the input type from its handler type - handler_type = type(spec.handler).__name__ + handler_type = type(trait_handler).__name__ + + # Deal with compound traits + # TODO create a mutually exclusive group for members of compound traits + if handler_type == "TraitCompound": + input_list = [] + # Recursively create an input for each trait + for i in range(0, len(trait_handler.handlers)): + input_list.append(get_boutiques_input(inputs, interface, input_name, spec, + ignored_template_inputs, verbose, + ignore_template_numbers, trait_handler.handlers[i], i)) + return input_list if handler_type == "File" or handler_type == "Directory": input['type'] = "File" @@ -155,19 +183,19 @@ def get_boutiques_input(inputs, interface, input_name, spec, # Deal with range inputs if handler_type == "Range": input['type'] = "Number" - if spec.handler.low is not None: - input['minimum'] = spec.handler.low - if spec.handler.high is not None: - input['maximum'] = spec.handler.high - if spec.handler.exclude_low is not None: - input['exclusive-minimum'] = spec.handler.exclude_low - if spec.handler.exclude_high is not None: - input['exclusive-maximum'] = spec.handler.exclude_high + if trait_handler.low is not None: + input['minimum'] = trait_handler.low + if trait_handler.high is not None: + input['maximum'] = trait_handler.high + if trait_handler.exclude_low is not None: + input['exclusive-minimum'] = trait_handler.exclude_low + if trait_handler.exclude_high is not None: + input['exclusive-maximum'] = trait_handler.exclude_high # Deal with list inputs if handler_type == "List": input['list'] = True - trait_type = type(spec.handler.item_trait.trait_type).__name__ + trait_type = type(trait_handler.item_trait.trait_type).__name__ if trait_type == "Int": input['integer'] = True input['type'] = "Number" @@ -175,10 +203,10 @@ def get_boutiques_input(inputs, interface, input_name, spec, input['type'] = "Number" else: input['type'] = "String" - if spec.handler.minlen is not None: - input['min-list-entries'] = spec.handler.minlen - if spec.handler.maxlen is not None: - input['max-list-entries'] = spec.handler.maxlen + if trait_handler.minlen is not None: + input['min-list-entries'] = trait_handler.minlen + if trait_handler.maxlen is not None: + input['max-list-entries'] = trait_handler.maxlen input['value-key'] = "[" + input_name.upper( ) + "]" # assumes that input names are unique @@ -200,7 +228,7 @@ def get_boutiques_input(inputs, interface, input_name, spec, input['default-value'] = spec.default_value()[1] try: - value_choices = spec.handler.values + value_choices = trait_handler.values except AttributeError: pass else: @@ -211,7 +239,10 @@ def get_boutiques_input(inputs, interface, input_name, spec, input['requires-inputs'] = spec.requires if spec.xor is not None: - input['disables-inputs'] = spec.xor + input['disables-inputs'] = list(spec.xor) + # Make sure input does not disable itself + if input['id'] in input['disables-inputs']: + input['disables-inputs'].remove(input['id']) # Create unique, temporary value. temp_value = must_generate_value(input_name, input['type'], @@ -304,11 +335,9 @@ def get_boutiques_output(outputs, name, spec, interface, tool_inputs, verbose=Fa output['path-template'] = input['value-key'] found = True break - # If no input with the same name was found, warn the user they should provide it manually + # If no input with the same name was found, use the output ID if not found: - print("WARNING: Could not determine path template for output %s. Please provide one for the " - "descriptor manually." % name) - output['path-template'] = "WARNING: No path template provided." + output['path-template'] = output['id'] return output diff --git a/nipype/utils/tests/test_nipype2boutiques.py b/nipype/utils/tests/test_nipype2boutiques.py index 3aecefda63..ac12c527f7 100644 --- a/nipype/utils/tests/test_nipype2boutiques.py +++ b/nipype/utils/tests/test_nipype2boutiques.py @@ -15,4 +15,5 @@ def test_generate(): container_index=None, container_type=None, verbose=False, - ignore_template_numbers=False) + ignore_template_numbers=False, + save=False) From ad36016e581703ea23d79122a5bfb5a7acb6c54c Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Mon, 11 Feb 2019 22:49:52 -0500 Subject: [PATCH 08/28] put compound inputs into mutex group --- nipype/utils/nipype2boutiques.py | 44 ++++++++++---------------------- 1 file changed, 13 insertions(+), 31 deletions(-) diff --git a/nipype/utils/nipype2boutiques.py b/nipype/utils/nipype2boutiques.py index 34b79437fa..2e29be8aba 100644 --- a/nipype/utils/nipype2boutiques.py +++ b/nipype/utils/nipype2boutiques.py @@ -78,12 +78,20 @@ def generate_boutiques_descriptor( input = get_boutiques_input(inputs, interface, name, spec, ignored_template_inputs, verbose, ignore_template_numbers) + # Handle compound inputs (inputs that can be of multiple types and are mutually exclusive) if isinstance(input, list): + mutex_group_members = [] for i in input: tool_desc['inputs'].append(i) tool_desc['command-line'] += i['value-key'] + " " + mutex_group_members.append(i['id']) if verbose: print("-> Adding input " + i['name']) + # Put inputs into a mutually exclusive group + tool_desc['groups'] = [{'id': input[0]['id'] + "_group", + 'name': input[0]['name'], + 'members': mutex_group_members, + 'mutually-exclusive': True}] else: tool_desc['inputs'].append(input) tool_desc['command-line'] += input['value-key'] + " " @@ -123,7 +131,8 @@ def generate_boutiques_descriptor( def get_boutiques_input(inputs, interface, input_name, spec, ignored_template_inputs, verbose, - ignore_template_numbers, handler=None, input_number=None): + ignore_template_numbers, handler=None, + input_number=None): """ Returns a dictionary containing the Boutiques input corresponding to a Nipype intput. @@ -134,6 +143,8 @@ def get_boutiques_input(inputs, interface, input_name, spec, * spec: Nipype input spec. * ignored_template_inputs: input names for which no temporary value must be generated. * ignore_template_numbers: True if numbers must be ignored in output path creations. + * handler: used when handling compound inputs, which don't have their own input spec + * input_number: used when handling compound inputs to assign each a unique ID Assumes that: * Input names are unique. @@ -142,7 +153,7 @@ def get_boutiques_input(inputs, interface, input_name, spec, input = {} - if input_number is not None: + if input_number is not None and input_number != 0: # No need to append a number to the first of a list of compound inputs input['id'] = input_name + "_" + str(input_number + 1) else: input['id'] = input_name @@ -158,7 +169,6 @@ def get_boutiques_input(inputs, interface, input_name, spec, handler_type = type(trait_handler).__name__ # Deal with compound traits - # TODO create a mutually exclusive group for members of compound traits if handler_type == "TraitCompound": input_list = [] # Recursively create an input for each trait @@ -341,34 +351,6 @@ def get_boutiques_output(outputs, name, spec, interface, tool_inputs, verbose=Fa return output -# TODO remove this -def get_type_from_spec_info(spec_info): - ''' - Returns an input type from the spec info. There must be a better - way to get an input type in Nipype than to parse the spec info. - ''' - if ("an existing file name" in spec_info) or ( - "input volumes" in spec_info): - return "File" - elif ("an integer" in spec_info or "a float" in spec_info): - return "Number" - elif "a boolean" in spec_info: - return "Boolean" - return "String" - - -# TODO remove this -def is_list(spec_info): - ''' - Returns True if the spec info looks like it describes a list - parameter. There must be a better way in Nipype to check if an input - is a list. - ''' - if "a list" in spec_info: - return True - return False - - def get_unique_value(type, id): ''' Returns a unique value of type 'type', for input with id 'id', From bb260f8459a3e8cb016c635f8cea9b7c3e20c4af Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Tue, 12 Feb 2019 15:04:58 -0500 Subject: [PATCH 09/28] added method to get all the mutex and all-or-none groups from the input specs --- nipype/utils/nipype2boutiques.py | 54 ++++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 13 deletions(-) diff --git a/nipype/utils/nipype2boutiques.py b/nipype/utils/nipype2boutiques.py index 2e29be8aba..1f46b24885 100644 --- a/nipype/utils/nipype2boutiques.py +++ b/nipype/utils/nipype2boutiques.py @@ -64,6 +64,7 @@ def generate_boutiques_descriptor( 'description'] = interface_name + ", as implemented in Nipype (module: " + module_name + ", interface: " + interface_name + ")." tool_desc['inputs'] = [] tool_desc['output-files'] = [] + tool_desc['groups'] = [] tool_desc['tool-version'] = interface.version if interface.version is not None else "No version provided." tool_desc['schema-version'] = '0.5' if container_image: @@ -88,16 +89,21 @@ def generate_boutiques_descriptor( if verbose: print("-> Adding input " + i['name']) # Put inputs into a mutually exclusive group - tool_desc['groups'] = [{'id': input[0]['id'] + "_group", - 'name': input[0]['name'], - 'members': mutex_group_members, - 'mutually-exclusive': True}] + tool_desc['groups'].append({'id': input[0]['id'] + "_group", + 'name': input[0]['name'] + " group", + 'members': mutex_group_members, + 'mutually-exclusive': True}) else: tool_desc['inputs'].append(input) tool_desc['command-line'] += input['value-key'] + " " if verbose: print("-> Adding input " + input['name']) + # Generates input groups + tool_desc['groups'] += get_boutiques_groups(interface.inputs.traits(transient=None).items()) + if len(tool_desc['groups']) == 0: + del tool_desc['groups'] + # Generates tool outputs for name, spec in sorted(outputs.traits(transient=None).items()): output = get_boutiques_output(outputs, name, spec, interface, tool_desc['inputs'], @@ -245,15 +251,6 @@ def get_boutiques_input(inputs, interface, input_name, spec, if value_choices is not None: input['value-choices'] = value_choices - if spec.requires is not None: - input['requires-inputs'] = spec.requires - - if spec.xor is not None: - input['disables-inputs'] = list(spec.xor) - # Make sure input does not disable itself - if input['id'] in input['disables-inputs']: - input['disables-inputs'].remove(input['id']) - # Create unique, temporary value. temp_value = must_generate_value(input_name, input['type'], ignored_template_inputs, spec_info, spec, @@ -351,6 +348,37 @@ def get_boutiques_output(outputs, name, spec, interface, tool_inputs, verbose=Fa return output +def get_boutiques_groups(input_traits): + desc_groups = [] + all_or_none_input_sets = [] + mutex_input_sets = [] + + # Get all the groups + for name, spec in input_traits: + if spec.requires is not None: + group_members = set([name] + list(spec.requires)) + if group_members not in all_or_none_input_sets: + all_or_none_input_sets.append(group_members) + if spec.xor is not None: + group_members = set([name] + list(spec.xor)) + if group_members not in mutex_input_sets: + mutex_input_sets.append(group_members) + + # Create a dictionary for each one + for i in range(0, len(all_or_none_input_sets)): + desc_groups.append({'id': "all_or_none_group" + ("_" + str(i + 1) if i != 0 else ""), + 'name': "All or none group" + (" " + str(i + 1) if i != 0 else ""), + 'members': list(all_or_none_input_sets[i]), + 'all-or-none': True}) + + for i in range(0, len(mutex_input_sets)): + desc_groups.append({'id': "mutex_group" + ("_" + str(i + 1) if i != 0 else ""), + 'name': "Mutex group" + (" " + str(i + 1) if i != 0 else ""), + 'members': list(mutex_input_sets[i]), + 'mutually-exclusive': True}) + + return desc_groups + def get_unique_value(type, id): ''' Returns a unique value of type 'type', for input with id 'id', From 1110b4fcf8e3a242661e55478753cd3488c5fb6b Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Mon, 25 Feb 2019 21:44:15 -0500 Subject: [PATCH 10/28] removed input tempvalue stuff and added logic to rerun the interface with various different inputs to try and generate as many outputs as possible --- nipype/utils/nipype2boutiques.py | 178 +++++++++++++++++++------------ 1 file changed, 109 insertions(+), 69 deletions(-) diff --git a/nipype/utils/nipype2boutiques.py b/nipype/utils/nipype2boutiques.py index 1f46b24885..54d43e94ec 100644 --- a/nipype/utils/nipype2boutiques.py +++ b/nipype/utils/nipype2boutiques.py @@ -25,18 +25,19 @@ def generate_boutiques_descriptor( - module, interface_name, ignored_template_inputs, container_image, - container_index, container_type, verbose, ignore_template_numbers, save): + module, interface_name, container_image, container_type, container_index=None, + ignored_template_inputs=(), ignore_template_numbers=False, verbose=False, save=False): ''' Returns a JSON string containing a JSON Boutiques description of a Nipype interface. Arguments: * module: module where the Nipype interface is declared. * interface_name: name of Nipype interface. - * ignored_template_inputs: a list of input names that should be ignored in the generation of output path templates. - * ignore_template_numbers: True if numbers must be ignored in output path creations. * container_image: name of the container image where the tool is installed - * container_index: optional index where the image is available * container_type: type of container image (Docker or Singularity) + * container_index: optional index where the image is available + * ignored_template_inputs: a list of input names that should be ignored in the generation of output path templates. + * ignore_template_numbers: True if numbers must be ignored in output path creations. + * verbose: print information messages * save: True if you want to save descriptor to a file ''' @@ -105,23 +106,21 @@ def generate_boutiques_descriptor( del tool_desc['groups'] # Generates tool outputs - for name, spec in sorted(outputs.traits(transient=None).items()): - output = get_boutiques_output(outputs, name, spec, interface, tool_desc['inputs'], - verbose) - if output['path-template'] != "": - tool_desc['output-files'].append(output) - if verbose: - print("-> Adding output " + output['name']) - elif verbose: - print("xx Skipping output " + output['name'] + - " with no path template.") - if tool_desc['output-files'] == []: - raise Exception("Tool has no output.") + generate_tool_outputs(outputs, interface, tool_desc, verbose, True) - # Removes all temporary values from inputs (otherwise they will - # appear in the JSON output) - for input in tool_desc['inputs']: - del input['tempvalue'] + # Generate outputs with various different inputs to try to generate + # as many output values as possible + custom_inputs = generate_custom_inputs(tool_desc['inputs']) + + for input_dict in custom_inputs: + interface = getattr(module, interface_name)(**input_dict) + outputs = interface.output_spec() + generate_tool_outputs(outputs, interface, tool_desc, verbose, False) + + # Fill in all missing output paths + for output in tool_desc['output-files']: + if output['path-template'] == "": + fill_in_missing_output_path(output, output['name'], tool_desc['inputs']) # Save descriptor to a file if save: @@ -135,6 +134,26 @@ def generate_boutiques_descriptor( return json.dumps(tool_desc, indent=4, separators=(',', ': ')) +def generate_tool_outputs(outputs, interface, tool_desc, verbose, first_run): + for name, spec in sorted(outputs.traits(transient=None).items()): + output = get_boutiques_output(outputs, name, spec, interface, tool_desc['inputs'], + verbose) + # If this is the first time we are generating outputs, add the full output to the descriptor. + # Otherwise, find the existing output and update its path template if it's still undefined. + if first_run: + tool_desc['output-files'].append(output) + else: + for existing_output in tool_desc['output-files']: + if output['id'] == existing_output['id'] and existing_output['path-template'] == "": + existing_output['path-template'] = output['path-template'] + break + if verbose: + print("-> Adding output " + output['name']) + + if len(tool_desc['output-files']) == 0: + raise Exception("Tool has no output.") + + def get_boutiques_input(inputs, interface, input_name, spec, ignored_template_inputs, verbose, ignore_template_numbers, handler=None, @@ -155,7 +174,6 @@ def get_boutiques_input(inputs, interface, input_name, spec, Assumes that: * Input names are unique. """ - spec_info = spec.full_info(inputs, input_name, None) input = {} @@ -234,7 +252,6 @@ def get_boutiques_input(inputs, interface, input_name, spec, elif input['type'] == "Flag": input['command-line-flag'] = ("--%s" % input_name + " ").strip() - input['tempvalue'] = None input['description'] = get_description_from_spec(inputs, input_name, spec) if not (hasattr(spec, "mandatory") and spec.mandatory): input['optional'] = True @@ -251,20 +268,7 @@ def get_boutiques_input(inputs, interface, input_name, spec, if value_choices is not None: input['value-choices'] = value_choices - # Create unique, temporary value. - temp_value = must_generate_value(input_name, input['type'], - ignored_template_inputs, spec_info, spec, - ignore_template_numbers) - if temp_value: - tempvalue = get_unique_value(input['type'], input_name) - setattr(interface.inputs, input_name, tempvalue) - input['tempvalue'] = tempvalue - if verbose: - print("oo Path-template creation using " + input['id'] + "=" + - str(tempvalue)) - - # Now that temp values have been generated, set Boolean types to - # Flag (there is no Boolean type in Boutiques) + # Set Boolean types to Flag (there is no Boolean type in Boutiques) if input['type'] == "Boolean": input['type'] = "Flag" @@ -313,38 +317,13 @@ def get_boutiques_output(outputs, name, spec, interface, tool_inputs, verbose=Fa except TypeError: output_value = None - # If output value is defined, use its basename - if output_value != "" and isinstance( - output_value, - str): # FIXME: this crashes when there are multiple output values. - # Go find from which input value it was built - for input in tool_inputs: - if not input['tempvalue']: - continue - input_value = input['tempvalue'] - if input['type'] == "File": - # Take the base name - input_value = os.path.splitext( - os.path.basename(input_value))[0] - if str(input_value) in output_value: - output_value = os.path.basename( - output_value.replace(input_value, - input['value-key']) - ) # FIXME: this only works if output is written in the current directory - output['path-template'] = os.path.basename(output_value) - - # If output value is undefined, create a placeholder for the path template - if not output_value: - # Look for an input with the same name and use this as the path template - found = False - for input in tool_inputs: - if input['id'] == name: - output['path-template'] = input['value-key'] - found = True - break - # If no input with the same name was found, use the output ID - if not found: - output['path-template'] = output['id'] + # If an output value is defined, use its relative path + # Otherwise, put blank string and try to fill it on another iteration + if output_value: + output['path-template'] = os.path.relpath(output_value) + else: + output['path-template'] = "" + return output @@ -379,6 +358,7 @@ def get_boutiques_groups(input_traits): return desc_groups + def get_unique_value(type, id): ''' Returns a unique value of type 'type', for input with id 'id', @@ -453,3 +433,63 @@ def get_description_from_spec(object, name, spec): boutiques_description += '.' return boutiques_description + + +def fill_in_missing_output_path(output, output_name, tool_inputs): + ''' + Creates a path template for outputs that are missing one + This is needed for the descriptor to be valid (path template is required) + ''' + # Look for an input with the same name as the output and use its value key + found = False + for input in tool_inputs: + if input['name'] == output_name: + output['path-template'] = input['value-key'] + found = True + break + # If no input with the same name was found, use the output ID + if not found: + output['path-template'] = output['id'] + return output + + + +def generate_custom_inputs(desc_inputs): + ''' + Generates a bunch of custom input dictionaries in order to generate as many outputs as possible + (to get their path templates) + Limitations: + -Does not support String inputs since some interfaces require specific strings + -Does not support File inputs since the file has to actually exist or the interface will fail + -Does not support list inputs yet + ''' + custom_input_dicts = [] + for desc_input in desc_inputs: + if desc_input.get('list'): # TODO support list inputs + continue + if desc_input['type'] == 'Flag': + custom_input_dicts.append({desc_input['id']: True}) + elif desc_input['type'] == 'Number': + custom_input_dicts.append({desc_input['id']: generate_random_number_input(desc_input)}) + elif desc_input.get('value-choices'): + for value in desc_input['value-choices']: + custom_input_dicts.append({desc_input['id']: value}) + return custom_input_dicts + + +def generate_random_number_input(desc_input): + ''' + Generates a random number input based on the input spec + ''' + if not desc_input.get('minimum') and not desc_input.get('maximum'): + return 1 + + if desc_input.get('integer'): + offset = 1 + else: + offset = 0.1 + + if desc_input.get('minimum'): + return desc_input['minimum'] if desc_input.get('exclusive-minimum') else desc_input['minimum'] + offset + if desc_input.get('maximum'): + return desc_input['maximum'] if desc_input.get('exclusive-maximum') else desc_input['maximum'] - offset From 8e20d2d142452c48e0bb6c9f876e2d29b9746e5f Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Mon, 25 Feb 2019 23:42:01 -0500 Subject: [PATCH 11/28] added logic to handle multioutputs and inputs where value choices are numbers, some other minor fixes --- nipype/utils/nipype2boutiques.py | 53 ++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/nipype/utils/nipype2boutiques.py b/nipype/utils/nipype2boutiques.py index 54d43e94ec..852887052f 100644 --- a/nipype/utils/nipype2boutiques.py +++ b/nipype/utils/nipype2boutiques.py @@ -20,6 +20,8 @@ import sys import tempfile import simplejson as json +import copy +import six from ..scripts.instance import import_module @@ -83,9 +85,9 @@ def generate_boutiques_descriptor( # Handle compound inputs (inputs that can be of multiple types and are mutually exclusive) if isinstance(input, list): mutex_group_members = [] + tool_desc['command-line'] += input[0]['value-key'] + " " for i in input: tool_desc['inputs'].append(i) - tool_desc['command-line'] += i['value-key'] + " " mutex_group_members.append(i['id']) if verbose: print("-> Adding input " + i['name']) @@ -100,6 +102,9 @@ def generate_boutiques_descriptor( if verbose: print("-> Adding input " + input['name']) + # Remove the extra space at the end of the command line + tool_desc['command-line'] = tool_desc['command-line'].strip() + # Generates input groups tool_desc['groups'] += get_boutiques_groups(interface.inputs.traits(transient=None).items()) if len(tool_desc['groups']) == 0: @@ -141,14 +146,20 @@ def generate_tool_outputs(outputs, interface, tool_desc, verbose, first_run): # If this is the first time we are generating outputs, add the full output to the descriptor. # Otherwise, find the existing output and update its path template if it's still undefined. if first_run: - tool_desc['output-files'].append(output) + if isinstance(output, list): + tool_desc['output-files'].extend(output) + if verbose: + print("-> Adding output " + output[0]['name']) + else: + tool_desc['output-files'].append(output) + if verbose: + print("-> Adding output " + output['name']) else: for existing_output in tool_desc['output-files']: - if output['id'] == existing_output['id'] and existing_output['path-template'] == "": + if not isinstance(output, list) and output['id'] == existing_output['id'] \ + and existing_output['path-template'] == "": existing_output['path-template'] = output['path-template'] break - if verbose: - print("-> Adding output " + output['name']) if len(tool_desc['output-files']) == 0: raise Exception("Tool has no output.") @@ -197,9 +208,11 @@ def get_boutiques_input(inputs, interface, input_name, spec, input_list = [] # Recursively create an input for each trait for i in range(0, len(trait_handler.handlers)): - input_list.append(get_boutiques_input(inputs, interface, input_name, spec, - ignored_template_inputs, verbose, - ignore_template_numbers, trait_handler.handlers[i], i)) + inp = get_boutiques_input(inputs, interface, input_name, spec, + ignored_template_inputs, verbose, + ignore_template_numbers, trait_handler.handlers[i], i) + inp['optional'] = True + input_list.append(inp) return input_list if handler_type == "File" or handler_type == "Directory": @@ -227,6 +240,7 @@ def get_boutiques_input(inputs, interface, input_name, spec, input['exclusive-maximum'] = trait_handler.exclude_high # Deal with list inputs + # TODO handle lists of lists (e.g. FSL ProbTrackX seed input) if handler_type == "List": input['list'] = True trait_type = type(trait_handler.item_trait.trait_type).__name__ @@ -235,11 +249,13 @@ def get_boutiques_input(inputs, interface, input_name, spec, input['type'] = "Number" elif trait_type == "Float": input['type'] = "Number" + elif trait_type == "File": + input['type'] = "File" else: input['type'] = "String" - if trait_handler.minlen is not None: + if trait_handler.minlen != 0: input['min-list-entries'] = trait_handler.minlen - if trait_handler.maxlen is not None: + if trait_handler.maxlen != six.MAXSIZE: input['max-list-entries'] = trait_handler.maxlen input['value-key'] = "[" + input_name.upper( @@ -266,6 +282,11 @@ def get_boutiques_input(inputs, interface, input_name, spec, pass else: if value_choices is not None: + if all(isinstance(n, int) for n in value_choices): + input['type'] = "Number" + input['integer'] = True + elif all(isinstance(n, float) for n in value_choices): + input['type'] = "Number" input['value-choices'] = value_choices # Set Boolean types to Flag (there is no Boolean type in Boutiques) @@ -316,6 +337,18 @@ def get_boutiques_output(outputs, name, spec, interface, tool_inputs, verbose=Fa output_value = interface._list_outputs()[name] except TypeError: output_value = None + except AttributeError: + output_value = None + + # Handle multi-outputs + if isinstance(output_value, list): + output_list = [] + for i in range(0, len(output_value)): + output_copy = copy.deepcopy(output) + output_copy['path-template'] = os.path.relpath(output_value[i]) + output_copy['id'] += ("_" + str(i+1)) if i > 0 else "" + output_list.append(output_copy) + return output_list # If an output value is defined, use its relative path # Otherwise, put blank string and try to fill it on another iteration From 61c836fcdf168afb8556244b4c36753213967153 Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Tue, 26 Feb 2019 14:22:46 -0500 Subject: [PATCH 12/28] fixed multioutput, added handling of multiinput, added option to supply custom path for saved file --- nipype/utils/nipype2boutiques.py | 44 ++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/nipype/utils/nipype2boutiques.py b/nipype/utils/nipype2boutiques.py index 852887052f..41492640bb 100644 --- a/nipype/utils/nipype2boutiques.py +++ b/nipype/utils/nipype2boutiques.py @@ -28,7 +28,7 @@ def generate_boutiques_descriptor( module, interface_name, container_image, container_type, container_index=None, - ignored_template_inputs=(), ignore_template_numbers=False, verbose=False, save=False): + ignored_template_inputs=(), ignore_template_numbers=False, verbose=False, save=False, save_path=None): ''' Returns a JSON string containing a JSON Boutiques description of a Nipype interface. Arguments: @@ -41,6 +41,7 @@ def generate_boutiques_descriptor( * ignore_template_numbers: True if numbers must be ignored in output path creations. * verbose: print information messages * save: True if you want to save descriptor to a file + * save_path: file path for the saved descriptor (defaults to name of the interface in current directory) ''' if not module: @@ -129,7 +130,8 @@ def generate_boutiques_descriptor( # Save descriptor to a file if save: - with open(interface_name + '.json', 'w') as outfile: + path = save_path if save_path is not None else os.path.join(os.getcwd(), interface_name + '.json') + with open(path, 'w') as outfile: json.dump(tool_desc, outfile, indent=4, separators=(',', ': ')) if verbose: print("-> Descriptor saved to file " + outfile.name) @@ -146,18 +148,12 @@ def generate_tool_outputs(outputs, interface, tool_desc, verbose, first_run): # If this is the first time we are generating outputs, add the full output to the descriptor. # Otherwise, find the existing output and update its path template if it's still undefined. if first_run: - if isinstance(output, list): - tool_desc['output-files'].extend(output) - if verbose: - print("-> Adding output " + output[0]['name']) - else: - tool_desc['output-files'].append(output) - if verbose: - print("-> Adding output " + output['name']) + tool_desc['output-files'].append(output) + if verbose: + print("-> Adding output " + output['name']) else: for existing_output in tool_desc['output-files']: - if not isinstance(output, list) and output['id'] == existing_output['id'] \ - and existing_output['path-template'] == "": + if output['id'] == existing_output['id'] and existing_output['path-template'] == "": existing_output['path-template'] = output['path-template'] break @@ -258,6 +254,11 @@ def get_boutiques_input(inputs, interface, input_name, spec, if trait_handler.maxlen != six.MAXSIZE: input['max-list-entries'] = trait_handler.maxlen + # Deal with multi-input + if handler_type == "InputMultiObject": + input['type'] = "File" + input['list'] = True + input['value-key'] = "[" + input_name.upper( ) + "]" # assumes that input names are unique @@ -342,13 +343,18 @@ def get_boutiques_output(outputs, name, spec, interface, tool_inputs, verbose=Fa # Handle multi-outputs if isinstance(output_value, list): - output_list = [] - for i in range(0, len(output_value)): - output_copy = copy.deepcopy(output) - output_copy['path-template'] = os.path.relpath(output_value[i]) - output_copy['id'] += ("_" + str(i+1)) if i > 0 else "" - output_list.append(output_copy) - return output_list + output['list'] = True + # Check if all extensions are the same + extensions = [] + for val in output_value: + extensions.append(os.path.splitext(val)[1]) + # If extensions all the same, set path template as wildcard + extension + # Otherwise just use a wildcard + if len(set(extensions)) == 1: + output['path-template'] = "*" + extensions[0] + else: + output['path-template'] = "*" + return output # If an output value is defined, use its relative path # Otherwise, put blank string and try to fill it on another iteration From f4c8867f88b0895c1531e961c6b693981d6ebcf7 Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Tue, 26 Feb 2019 15:56:35 -0500 Subject: [PATCH 13/28] fixed number input min and max, removed numbers from custom inputs (caused too many failures) --- nipype/utils/nipype2boutiques.py | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/nipype/utils/nipype2boutiques.py b/nipype/utils/nipype2boutiques.py index 41492640bb..8ff2d11880 100644 --- a/nipype/utils/nipype2boutiques.py +++ b/nipype/utils/nipype2boutiques.py @@ -226,14 +226,14 @@ def get_boutiques_input(inputs, interface, input_name, spec, # Deal with range inputs if handler_type == "Range": input['type'] = "Number" - if trait_handler.low is not None: - input['minimum'] = trait_handler.low - if trait_handler.high is not None: - input['maximum'] = trait_handler.high - if trait_handler.exclude_low is not None: - input['exclusive-minimum'] = trait_handler.exclude_low - if trait_handler.exclude_high is not None: - input['exclusive-maximum'] = trait_handler.exclude_high + if trait_handler._low is not None: + input['minimum'] = trait_handler._low + if trait_handler._high is not None: + input['maximum'] = trait_handler._high + if trait_handler._exclude_low: + input['exclusive-minimum'] = True + if trait_handler._exclude_high: + input['exclusive-maximum'] = True # Deal with list inputs # TODO handle lists of lists (e.g. FSL ProbTrackX seed input) @@ -492,24 +492,16 @@ def fill_in_missing_output_path(output, output_name, tool_inputs): return output - def generate_custom_inputs(desc_inputs): ''' Generates a bunch of custom input dictionaries in order to generate as many outputs as possible (to get their path templates) - Limitations: - -Does not support String inputs since some interfaces require specific strings - -Does not support File inputs since the file has to actually exist or the interface will fail - -Does not support list inputs yet + Currently only works with flag inputs and inputs with defined value choices. ''' custom_input_dicts = [] for desc_input in desc_inputs: - if desc_input.get('list'): # TODO support list inputs - continue if desc_input['type'] == 'Flag': custom_input_dicts.append({desc_input['id']: True}) - elif desc_input['type'] == 'Number': - custom_input_dicts.append({desc_input['id']: generate_random_number_input(desc_input)}) elif desc_input.get('value-choices'): for value in desc_input['value-choices']: custom_input_dicts.append({desc_input['id']: value}) From 6961f02bcb1ba0e85c7de6a85ba34d74755f3dc1 Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Tue, 26 Feb 2019 16:39:03 -0500 Subject: [PATCH 14/28] removed unused things, updated comments --- nipype/utils/nipype2boutiques.py | 112 ++------------------ nipype/utils/tests/test_nipype2boutiques.py | 2 - 2 files changed, 11 insertions(+), 103 deletions(-) diff --git a/nipype/utils/nipype2boutiques.py b/nipype/utils/nipype2boutiques.py index 8ff2d11880..ff389fdb52 100644 --- a/nipype/utils/nipype2boutiques.py +++ b/nipype/utils/nipype2boutiques.py @@ -7,20 +7,12 @@ # Boutiques tools can be imported in CBRAIN (https://github.com/aces/cbrain) among other platforms. # # Limitations: -# * List outputs are not supported. -# * Default values are not extracted from the documentation of the Nipype interface. -# * The following input types must be ignored for the output path template creation (see option -t): -# ** String restrictions, i.e. String inputs that accept only a restricted set of values. -# ** mutually exclusive inputs. -# * Path-templates are wrong when output files are not created in the execution directory (e.g. when a sub-directory is created). -# * Optional outputs, i.e. outputs that not always produced, may not be detected. +# * Optional outputs, i.e. outputs that not always produced, may not be detected. They will, however, still be listed +# with a placeholder for the path template (either a value key or the output ID) that should be verified and corrected. import os -import argparse import sys -import tempfile import simplejson as json -import copy import six from ..scripts.instance import import_module @@ -28,7 +20,7 @@ def generate_boutiques_descriptor( module, interface_name, container_image, container_type, container_index=None, - ignored_template_inputs=(), ignore_template_numbers=False, verbose=False, save=False, save_path=None): + verbose=False, save=False, save_path=None): ''' Returns a JSON string containing a JSON Boutiques description of a Nipype interface. Arguments: @@ -37,8 +29,6 @@ def generate_boutiques_descriptor( * container_image: name of the container image where the tool is installed * container_type: type of container image (Docker or Singularity) * container_index: optional index where the image is available - * ignored_template_inputs: a list of input names that should be ignored in the generation of output path templates. - * ignore_template_numbers: True if numbers must be ignored in output path creations. * verbose: print information messages * save: True if you want to save descriptor to a file * save_path: file path for the saved descriptor (defaults to name of the interface in current directory) @@ -80,9 +70,7 @@ def generate_boutiques_descriptor( # Generates tool inputs for name, spec in sorted(interface.inputs.traits(transient=None).items()): - input = get_boutiques_input(inputs, interface, name, spec, - ignored_template_inputs, verbose, - ignore_template_numbers) + input = get_boutiques_input(inputs, interface, name, spec, verbose) # Handle compound inputs (inputs that can be of multiple types and are mutually exclusive) if isinstance(input, list): mutex_group_members = [] @@ -162,9 +150,7 @@ def generate_tool_outputs(outputs, interface, tool_desc, verbose, first_run): def get_boutiques_input(inputs, interface, input_name, spec, - ignored_template_inputs, verbose, - ignore_template_numbers, handler=None, - input_number=None): + verbose, handler=None, input_number=None): """ Returns a dictionary containing the Boutiques input corresponding to a Nipype intput. @@ -173,8 +159,7 @@ def get_boutiques_input(inputs, interface, input_name, spec, * interface: Nipype interface. * input_name: name of the Nipype input. * spec: Nipype input spec. - * ignored_template_inputs: input names for which no temporary value must be generated. - * ignore_template_numbers: True if numbers must be ignored in output path creations. + * verbose: print information messages. * handler: used when handling compound inputs, which don't have their own input spec * input_number: used when handling compound inputs to assign each a unique ID @@ -205,8 +190,7 @@ def get_boutiques_input(inputs, interface, input_name, spec, # Recursively create an input for each trait for i in range(0, len(trait_handler.handlers)): inp = get_boutiques_input(inputs, interface, input_name, spec, - ignored_template_inputs, verbose, - ignore_template_numbers, trait_handler.handlers[i], i) + verbose, trait_handler.handlers[i], i) inp['optional'] = True input_list.append(inp) return input_list @@ -297,7 +281,7 @@ def get_boutiques_input(inputs, interface, input_name, spec, return input -def get_boutiques_output(outputs, name, spec, interface, tool_inputs, verbose=False): +def get_boutiques_output(outputs, name, spec, interface, tool_inputs): """ Returns a dictionary containing the Boutiques output corresponding to a Nipype output. @@ -367,6 +351,9 @@ def get_boutiques_output(outputs, name, spec, interface, tool_inputs, verbose=Fa def get_boutiques_groups(input_traits): + """ + Returns a list of dictionaries containing Boutiques groups for the mutually exclusive and all-or-none Nipype inputs. + """ desc_groups = [] all_or_none_input_sets = [] mutex_input_sets = [] @@ -398,65 +385,6 @@ def get_boutiques_groups(input_traits): return desc_groups -def get_unique_value(type, id): - ''' - Returns a unique value of type 'type', for input with id 'id', - assuming id is unique. - ''' - return { - "File": os.path.abspath(create_tempfile()), - "Boolean": True, - "Number": abs(hash(id)), # abs in case input param must be positive... - "String": id - }[type] - - -def create_tempfile(): - ''' - Creates a temp file and returns its name. - ''' - fileTemp = tempfile.NamedTemporaryFile(delete=False) - fileTemp.write(b"hello") - fileTemp.close() - return fileTemp.name - - -def must_generate_value(name, type, ignored_template_inputs, spec_info, spec, - ignore_template_numbers): - ''' - Return True if a temporary value must be generated for this input. - Arguments: - * name: input name. - * type: input_type. - * ignored_template_inputs: a list of inputs names for which no value must be generated. - * spec_info: spec info of the Nipype input - * ignore_template_numbers: True if numbers must be ignored. - ''' - # Return false when type is number and numbers must be ignored. - if ignore_template_numbers and type == "Number": - return False - # Only generate value for the first element of mutually exclusive inputs. - if spec.xor and spec.xor[0] != name: - return False - # Directory types are not supported - if "an existing directory name" in spec_info: - return False - # Don't know how to generate a list. - if "a list" in spec_info or "a tuple" in spec_info: - return False - # Don't know how to generate a dictionary. - if "a dictionary" in spec_info: - return False - # Best guess to detect string restrictions... - if "' or '" in spec_info: - return False - if spec.default or spec.default_value(): - return False - if not ignored_template_inputs: - return True - return not (name in ignored_template_inputs) - - def get_description_from_spec(object, name, spec): ''' Generates a description based on the input or output spec. @@ -506,21 +434,3 @@ def generate_custom_inputs(desc_inputs): for value in desc_input['value-choices']: custom_input_dicts.append({desc_input['id']: value}) return custom_input_dicts - - -def generate_random_number_input(desc_input): - ''' - Generates a random number input based on the input spec - ''' - if not desc_input.get('minimum') and not desc_input.get('maximum'): - return 1 - - if desc_input.get('integer'): - offset = 1 - else: - offset = 0.1 - - if desc_input.get('minimum'): - return desc_input['minimum'] if desc_input.get('exclusive-minimum') else desc_input['minimum'] + offset - if desc_input.get('maximum'): - return desc_input['maximum'] if desc_input.get('exclusive-maximum') else desc_input['maximum'] - offset diff --git a/nipype/utils/tests/test_nipype2boutiques.py b/nipype/utils/tests/test_nipype2boutiques.py index ac12c527f7..926de0f2cd 100644 --- a/nipype/utils/tests/test_nipype2boutiques.py +++ b/nipype/utils/tests/test_nipype2boutiques.py @@ -10,10 +10,8 @@ def test_generate(): generate_boutiques_descriptor(module='nipype.interfaces.ants.registration', interface_name='ANTS', - ignored_template_inputs=(), container_image=None, container_index=None, container_type=None, verbose=False, - ignore_template_numbers=False, save=False) From f67d04b4b1c45728ca1961f8c11b859c35f5a852 Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Wed, 27 Feb 2019 19:20:43 -0500 Subject: [PATCH 15/28] fixed small error --- nipype/utils/nipype2boutiques.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nipype/utils/nipype2boutiques.py b/nipype/utils/nipype2boutiques.py index ff389fdb52..aba64c4414 100644 --- a/nipype/utils/nipype2boutiques.py +++ b/nipype/utils/nipype2boutiques.py @@ -131,8 +131,7 @@ def generate_boutiques_descriptor( def generate_tool_outputs(outputs, interface, tool_desc, verbose, first_run): for name, spec in sorted(outputs.traits(transient=None).items()): - output = get_boutiques_output(outputs, name, spec, interface, tool_desc['inputs'], - verbose) + output = get_boutiques_output(outputs, name, spec, interface, tool_desc['inputs']) # If this is the first time we are generating outputs, add the full output to the descriptor. # Otherwise, find the existing output and update its path template if it's still undefined. if first_run: From 2bf03cf1d6287984da2606cadde30d12e7408102 Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Tue, 12 Mar 2019 16:20:32 -0400 Subject: [PATCH 16/28] added author and ignored inputs parameters, removed nipype from command line, added logic to deal with inputs containing name_source and name_template metadata --- nipype/utils/nipype2boutiques.py | 53 +++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/nipype/utils/nipype2boutiques.py b/nipype/utils/nipype2boutiques.py index aba64c4414..0472673469 100644 --- a/nipype/utils/nipype2boutiques.py +++ b/nipype/utils/nipype2boutiques.py @@ -20,7 +20,7 @@ def generate_boutiques_descriptor( module, interface_name, container_image, container_type, container_index=None, - verbose=False, save=False, save_path=None): + verbose=False, save=False, save_path=None, author=None, ignore_inputs=None): ''' Returns a JSON string containing a JSON Boutiques description of a Nipype interface. Arguments: @@ -32,6 +32,8 @@ def generate_boutiques_descriptor( * verbose: print information messages * save: True if you want to save descriptor to a file * save_path: file path for the saved descriptor (defaults to name of the interface in current directory) + * author: author of the tool (required for publishing) + * ignore_inputs: list of interface inputs to not include in the descriptor ''' if not module: @@ -53,7 +55,10 @@ def generate_boutiques_descriptor( tool_desc = {} tool_desc['name'] = interface_name tool_desc[ - 'command-line'] = "nipype_cmd " + module_name + " " + interface_name + " " + 'command-line'] = interface_name + " " + tool_desc['author'] = "Nipype (interface)" + if author is not None: + tool_desc['author'] = tool_desc['author'] + ", " + author + " (tool)" tool_desc[ 'description'] = interface_name + ", as implemented in Nipype (module: " + module_name + ", interface: " + interface_name + ")." tool_desc['inputs'] = [] @@ -70,8 +75,10 @@ def generate_boutiques_descriptor( # Generates tool inputs for name, spec in sorted(interface.inputs.traits(transient=None).items()): - input = get_boutiques_input(inputs, interface, name, spec, verbose) + input = get_boutiques_input(inputs, interface, name, spec, verbose, ignore_inputs=ignore_inputs) # Handle compound inputs (inputs that can be of multiple types and are mutually exclusive) + if input is None: + continue if isinstance(input, list): mutex_group_members = [] tool_desc['command-line'] += input[0]['value-key'] + " " @@ -91,9 +98,6 @@ def generate_boutiques_descriptor( if verbose: print("-> Adding input " + input['name']) - # Remove the extra space at the end of the command line - tool_desc['command-line'] = tool_desc['command-line'].strip() - # Generates input groups tool_desc['groups'] += get_boutiques_groups(interface.inputs.traits(transient=None).items()) if len(tool_desc['groups']) == 0: @@ -116,6 +120,9 @@ def generate_boutiques_descriptor( if output['path-template'] == "": fill_in_missing_output_path(output, output['name'], tool_desc['inputs']) + # Remove the extra space at the end of the command line + tool_desc['command-line'] = tool_desc['command-line'].strip() + # Save descriptor to a file if save: path = save_path if save_path is not None else os.path.join(os.getcwd(), interface_name + '.json') @@ -136,6 +143,8 @@ def generate_tool_outputs(outputs, interface, tool_desc, verbose, first_run): # Otherwise, find the existing output and update its path template if it's still undefined. if first_run: tool_desc['output-files'].append(output) + if output.get('value-key'): + tool_desc['command-line'] += output['value-key'] + " " if verbose: print("-> Adding output " + output['name']) else: @@ -143,13 +152,15 @@ def generate_tool_outputs(outputs, interface, tool_desc, verbose, first_run): if output['id'] == existing_output['id'] and existing_output['path-template'] == "": existing_output['path-template'] = output['path-template'] break + if output.get('value-key') and output['value-key'] not in tool_desc['command-line']: + tool_desc['command-line'] += output['value-key'] + " " if len(tool_desc['output-files']) == 0: raise Exception("Tool has no output.") -def get_boutiques_input(inputs, interface, input_name, spec, - verbose, handler=None, input_number=None): +def get_boutiques_input(inputs, interface, input_name, spec, verbose, handler=None, + input_number=None, ignore_inputs=None): """ Returns a dictionary containing the Boutiques input corresponding to a Nipype intput. @@ -161,11 +172,17 @@ def get_boutiques_input(inputs, interface, input_name, spec, * verbose: print information messages. * handler: used when handling compound inputs, which don't have their own input spec * input_number: used when handling compound inputs to assign each a unique ID + * ignore_inputs: list of interface inputs to not include in the descriptor Assumes that: * Input names are unique. """ + # If spec has a name source, means it's an output, so skip it here. + # Also skip any ignored inputs + if spec.name_source or ignore_inputs is not None and input_name in ignore_inputs: + return None + input = {} if input_number is not None and input_number != 0: # No need to append a number to the first of a list of compound inputs @@ -339,12 +356,26 @@ def get_boutiques_output(outputs, name, spec, interface, tool_inputs): output['path-template'] = "*" return output - # If an output value is defined, use its relative path - # Otherwise, put blank string and try to fill it on another iteration + # If an output value is defined, use its relative path, if one exists. + # If no relative path, look for an input with the same name containing a name source + # and name template. Otherwise, put blank string as placeholder and try to fill it on + # another iteration. + output['path-template'] = "" + if output_value: output['path-template'] = os.path.relpath(output_value) else: - output['path-template'] = "" + for inp_name, inp_spec in sorted(interface.inputs.traits(transient=None).items()): + if inp_name == name and inp_spec.name_source and inp_spec.name_template: + if isinstance(inp_spec.name_source, list): + source = inp_spec.name_source[0] + else: + source = inp_spec.name_source + output['path-template'] = inp_spec.name_template.replace("%s", "[" + source.upper() + "]") + output['value-key'] = "[" + name.upper() + "]" + if inp_spec.argstr and inp_spec.argstr.split("%")[0]: + output['command-line-flag'] = inp_spec.argstr.split("%")[0].strip() + break return output From d009a88ac46e8971bf727f1221f18808014f74be Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Tue, 12 Mar 2019 16:57:10 -0400 Subject: [PATCH 17/28] fixed code quality issues --- nipype/utils/nipype2boutiques.py | 120 +++++++++++++++---------------- 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/nipype/utils/nipype2boutiques.py b/nipype/utils/nipype2boutiques.py index 0472673469..e5b91b0ea7 100644 --- a/nipype/utils/nipype2boutiques.py +++ b/nipype/utils/nipype2boutiques.py @@ -75,28 +75,28 @@ def generate_boutiques_descriptor( # Generates tool inputs for name, spec in sorted(interface.inputs.traits(transient=None).items()): - input = get_boutiques_input(inputs, interface, name, spec, verbose, ignore_inputs=ignore_inputs) + inp = get_boutiques_input(inputs, interface, name, spec, verbose, ignore_inputs=ignore_inputs) # Handle compound inputs (inputs that can be of multiple types and are mutually exclusive) - if input is None: + if inp is None: continue - if isinstance(input, list): + if isinstance(inp, list): mutex_group_members = [] - tool_desc['command-line'] += input[0]['value-key'] + " " - for i in input: + tool_desc['command-line'] += inp[0]['value-key'] + " " + for i in inp: tool_desc['inputs'].append(i) mutex_group_members.append(i['id']) if verbose: print("-> Adding input " + i['name']) # Put inputs into a mutually exclusive group - tool_desc['groups'].append({'id': input[0]['id'] + "_group", - 'name': input[0]['name'] + " group", + tool_desc['groups'].append({'id': inp[0]['id'] + "_group", + 'name': inp[0]['name'] + " group", 'members': mutex_group_members, 'mutually-exclusive': True}) else: - tool_desc['inputs'].append(input) - tool_desc['command-line'] += input['value-key'] + " " + tool_desc['inputs'].append(inp) + tool_desc['command-line'] += inp['value-key'] + " " if verbose: - print("-> Adding input " + input['name']) + print("-> Adding input " + inp['name']) # Generates input groups tool_desc['groups'] += get_boutiques_groups(interface.inputs.traits(transient=None).items()) @@ -183,14 +183,14 @@ def get_boutiques_input(inputs, interface, input_name, spec, verbose, handler=No if spec.name_source or ignore_inputs is not None and input_name in ignore_inputs: return None - input = {} + inp = {} if input_number is not None and input_number != 0: # No need to append a number to the first of a list of compound inputs - input['id'] = input_name + "_" + str(input_number + 1) + inp['id'] = input_name + "_" + str(input_number + 1) else: - input['id'] = input_name + inp['id'] = input_name - input['name'] = input_name.replace('_', ' ').capitalize() + inp['name'] = input_name.replace('_', ' ').capitalize() if handler is None: trait_handler = spec.handler @@ -212,70 +212,70 @@ def get_boutiques_input(inputs, interface, input_name, spec, verbose, handler=No return input_list if handler_type == "File" or handler_type == "Directory": - input['type'] = "File" + inp['type'] = "File" elif handler_type == "Int": - input['type'] = "Number" - input['integer'] = True + inp['type'] = "Number" + inp['integer'] = True elif handler_type == "Float": - input['type'] = "Number" + inp['type'] = "Number" elif handler_type == "Bool": - input['type'] = "Flag" + inp['type'] = "Flag" else: - input['type'] = "String" + inp['type'] = "String" # Deal with range inputs if handler_type == "Range": - input['type'] = "Number" + inp['type'] = "Number" if trait_handler._low is not None: - input['minimum'] = trait_handler._low + inp['minimum'] = trait_handler._low if trait_handler._high is not None: - input['maximum'] = trait_handler._high + inp['maximum'] = trait_handler._high if trait_handler._exclude_low: - input['exclusive-minimum'] = True + inp['exclusive-minimum'] = True if trait_handler._exclude_high: - input['exclusive-maximum'] = True + inp['exclusive-maximum'] = True # Deal with list inputs # TODO handle lists of lists (e.g. FSL ProbTrackX seed input) if handler_type == "List": - input['list'] = True + inp['list'] = True trait_type = type(trait_handler.item_trait.trait_type).__name__ if trait_type == "Int": - input['integer'] = True - input['type'] = "Number" + inp['integer'] = True + inp['type'] = "Number" elif trait_type == "Float": - input['type'] = "Number" + inp['type'] = "Number" elif trait_type == "File": - input['type'] = "File" + inp['type'] = "File" else: - input['type'] = "String" + inp['type'] = "String" if trait_handler.minlen != 0: - input['min-list-entries'] = trait_handler.minlen + inp['min-list-entries'] = trait_handler.minlen if trait_handler.maxlen != six.MAXSIZE: - input['max-list-entries'] = trait_handler.maxlen + inp['max-list-entries'] = trait_handler.maxlen # Deal with multi-input if handler_type == "InputMultiObject": - input['type'] = "File" - input['list'] = True + inp['type'] = "File" + inp['list'] = True - input['value-key'] = "[" + input_name.upper( + inp['value-key'] = "[" + input_name.upper( ) + "]" # assumes that input names are unique # Add the command line flag specified by argstr # If no argstr is provided and input type is Flag, create a flag from the name if spec.argstr and spec.argstr.split("%")[0]: - input['command-line-flag'] = spec.argstr.split("%")[0].strip() - elif input['type'] == "Flag": - input['command-line-flag'] = ("--%s" % input_name + " ").strip() + inp['command-line-flag'] = spec.argstr.split("%")[0].strip() + elif inp['type'] == "Flag": + inp['command-line-flag'] = ("--%s" % input_name + " ").strip() - input['description'] = get_description_from_spec(inputs, input_name, spec) + inp['description'] = get_description_from_spec(inputs, input_name, spec) if not (hasattr(spec, "mandatory") and spec.mandatory): - input['optional'] = True + inp['optional'] = True else: - input['optional'] = False + inp['optional'] = False if spec.usedefault: - input['default-value'] = spec.default_value()[1] + inp['default-value'] = spec.default_value()[1] try: value_choices = trait_handler.values @@ -284,17 +284,17 @@ def get_boutiques_input(inputs, interface, input_name, spec, verbose, handler=No else: if value_choices is not None: if all(isinstance(n, int) for n in value_choices): - input['type'] = "Number" - input['integer'] = True + inp['type'] = "Number" + inp['integer'] = True elif all(isinstance(n, float) for n in value_choices): - input['type'] = "Number" - input['value-choices'] = value_choices + inp['type'] = "Number" + inp['value-choices'] = value_choices # Set Boolean types to Flag (there is no Boolean type in Boutiques) - if input['type'] == "Boolean": - input['type'] = "Flag" + if inp['type'] == "Boolean": + inp['type'] = "Flag" - return input + return inp def get_boutiques_output(outputs, name, spec, interface, tool_inputs): @@ -400,28 +400,28 @@ def get_boutiques_groups(input_traits): mutex_input_sets.append(group_members) # Create a dictionary for each one - for i in range(0, len(all_or_none_input_sets)): - desc_groups.append({'id': "all_or_none_group" + ("_" + str(i + 1) if i != 0 else ""), - 'name': "All or none group" + (" " + str(i + 1) if i != 0 else ""), - 'members': list(all_or_none_input_sets[i]), + for i, inp_set in enumerate(all_or_none_input_sets, 1): + desc_groups.append({'id': "all_or_none_group" + ("_" + str(i) if i != 1 else ""), + 'name': "All or none group" + (" " + str(i) if i != 1 else ""), + 'members': list(inp_set), 'all-or-none': True}) - for i in range(0, len(mutex_input_sets)): - desc_groups.append({'id': "mutex_group" + ("_" + str(i + 1) if i != 0 else ""), - 'name': "Mutex group" + (" " + str(i + 1) if i != 0 else ""), - 'members': list(mutex_input_sets[i]), + for i, inp_set in enumerate(mutex_input_sets, 1): + desc_groups.append({'id': "mutex_group" + ("_" + str(i) if i != 1 else ""), + 'name': "Mutex group" + (" " + str(i) if i != 1 else ""), + 'members': list(inp_set), 'mutually-exclusive': True}) return desc_groups -def get_description_from_spec(object, name, spec): +def get_description_from_spec(obj, name, spec): ''' Generates a description based on the input or output spec. ''' if not spec.desc: spec.desc = "No description provided." - spec_info = spec.full_info(object, name, None) + spec_info = spec.full_info(obj, name, None) boutiques_description = (spec_info.capitalize( ) + ". " + spec.desc.capitalize()).replace("\n", '') From 63d96be755b3f550fc3db97ff2ac4272b54825ff Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Mon, 18 Mar 2019 17:17:14 -0400 Subject: [PATCH 18/28] added option to supply descriptor tags, changed default tool version to 1.0.0, added logic to deal with 0/1 booleans and tuples, added list separator and flag separator checks --- nipype/utils/nipype2boutiques.py | 59 +++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/nipype/utils/nipype2boutiques.py b/nipype/utils/nipype2boutiques.py index e5b91b0ea7..6f2988c4ba 100644 --- a/nipype/utils/nipype2boutiques.py +++ b/nipype/utils/nipype2boutiques.py @@ -20,7 +20,7 @@ def generate_boutiques_descriptor( module, interface_name, container_image, container_type, container_index=None, - verbose=False, save=False, save_path=None, author=None, ignore_inputs=None): + verbose=False, save=False, save_path=None, author=None, ignore_inputs=None, tags=None): ''' Returns a JSON string containing a JSON Boutiques description of a Nipype interface. Arguments: @@ -34,6 +34,8 @@ def generate_boutiques_descriptor( * save_path: file path for the saved descriptor (defaults to name of the interface in current directory) * author: author of the tool (required for publishing) * ignore_inputs: list of interface inputs to not include in the descriptor + * tags: JSON object containing tags to include in the descriptor, e.g. "{/"key1/": /"value1/"}" + (note: the tags 'domain:neuroinformatics' and 'interface-type:nipype' are included by default) ''' if not module: @@ -64,7 +66,7 @@ def generate_boutiques_descriptor( tool_desc['inputs'] = [] tool_desc['output-files'] = [] tool_desc['groups'] = [] - tool_desc['tool-version'] = interface.version if interface.version is not None else "No version provided." + tool_desc['tool-version'] = interface.version if interface.version is not None else "1.0.0" tool_desc['schema-version'] = '0.5' if container_image: tool_desc['container-image'] = {} @@ -120,6 +122,24 @@ def generate_boutiques_descriptor( if output['path-template'] == "": fill_in_missing_output_path(output, output['name'], tool_desc['inputs']) + # Add tags + desc_tags = { + 'domain': 'neuroinformatics', + 'source': 'nipype-interface' + } + + if tags is not None: + tags_dict = json.loads(tags) + for k, v in tags_dict.items(): + if k in desc_tags: + if not isinstance(desc_tags[k], list): + desc_tags[k] = [desc_tags[k]] + desc_tags[k].append(v) + else: + desc_tags[k] = v + + tool_desc['tags'] = desc_tags + # Remove the extra space at the end of the command line tool_desc['command-line'] = tool_desc['command-line'].strip() @@ -219,7 +239,13 @@ def get_boutiques_input(inputs, interface, input_name, spec, verbose, handler=No elif handler_type == "Float": inp['type'] = "Number" elif handler_type == "Bool": - inp['type'] = "Flag" + if spec.argstr and len(spec.argstr.split("=")) > 1 and (spec.argstr.split("=")[1] == '0' or spec.argstr.split("=")[1] == '1'): + inp['type'] = "Number" + inp['integer'] = True + inp['minimum'] = 0 + inp['maximum'] = 1 + else: + inp['type'] = "Flag" else: inp['type'] = "String" @@ -253,6 +279,21 @@ def get_boutiques_input(inputs, interface, input_name, spec, verbose, handler=No inp['min-list-entries'] = trait_handler.minlen if trait_handler.maxlen != six.MAXSIZE: inp['max-list-entries'] = trait_handler.maxlen + if spec.sep: + inp['list-separator'] = spec.sep + + if handler_type == "Tuple": + inp['list'] = True + inp['min-list-entries'] = len(spec.default) + inp['max-list-entries'] = len(spec.default) + input_type = type(spec.default[0]).__name__ + if input_type == 'int': + inp['type'] = "Number" + inp['integer'] = True + elif input_type == 'float': + inp['type'] = "Number" + else: + inp['type'] = "String" # Deal with multi-input if handler_type == "InputMultiObject": @@ -264,8 +305,12 @@ def get_boutiques_input(inputs, interface, input_name, spec, verbose, handler=No # Add the command line flag specified by argstr # If no argstr is provided and input type is Flag, create a flag from the name - if spec.argstr and spec.argstr.split("%")[0]: - inp['command-line-flag'] = spec.argstr.split("%")[0].strip() + if spec.argstr: + if "=" in spec.argstr: + inp['command-line-flag'] = spec.argstr.split("=")[0].strip() + inp['command-line-flag-separator'] = "=" + elif spec.argstr.split("%")[0]: + inp['command-line-flag'] = spec.argstr.split("%")[0].strip() elif inp['type'] == "Flag": inp['command-line-flag'] = ("--%s" % input_name + " ").strip() @@ -290,10 +335,6 @@ def get_boutiques_input(inputs, interface, input_name, spec, verbose, handler=No inp['type'] = "Number" inp['value-choices'] = value_choices - # Set Boolean types to Flag (there is no Boolean type in Boutiques) - if inp['type'] == "Boolean": - inp['type'] = "Flag" - return inp From 1e15b9989f2498e698294532c1a0703596c7b3e2 Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Tue, 19 Mar 2019 22:48:42 -0400 Subject: [PATCH 19/28] added method to take into account the order of positional command line args --- nipype/utils/nipype2boutiques.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/nipype/utils/nipype2boutiques.py b/nipype/utils/nipype2boutiques.py index 6f2988c4ba..c5280d2d45 100644 --- a/nipype/utils/nipype2boutiques.py +++ b/nipype/utils/nipype2boutiques.py @@ -140,6 +140,9 @@ def generate_boutiques_descriptor( tool_desc['tags'] = desc_tags + # Check for positional arguments and reorder command line args if necessary + tool_desc['command-line'] = reorder_cmd_line_args(tool_desc['command-line'], interface, ignore_inputs) + # Remove the extra space at the end of the command line tool_desc['command-line'] = tool_desc['command-line'].strip() @@ -505,3 +508,31 @@ def generate_custom_inputs(desc_inputs): for value in desc_input['value-choices']: custom_input_dicts.append({desc_input['id']: value}) return custom_input_dicts + + +def reorder_cmd_line_args(cmd_line, interface, ignore_inputs=None): + ''' + Generates a new command line with the positional arguments in the correct order + ''' + interface_name = cmd_line.split()[0] + positional_arg_dict = {} + positional_args = [] + non_positional_args = [] + + for name, spec in sorted(interface.inputs.traits(transient=None).items()): + if ignore_inputs is not None and name in ignore_inputs: + continue + value_key = "[" + name.upper() + "]" + if spec.position is not None: + positional_arg_dict[spec.position] = value_key + else: + non_positional_args.append(value_key) + + last_arg = None + for item in sorted(positional_arg_dict.items()): + if item[0] == -1: + last_arg = item[1] + continue + positional_args.append(item[1]) + + return interface_name + " " + " ".join(positional_args) + " " + ((last_arg + " ") if last_arg else "") + " ".join(non_positional_args) From f0a11b835ede75463ee50745896d60afc0d962b2 Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Wed, 27 Mar 2019 17:02:40 -0400 Subject: [PATCH 20/28] added methods to generate command line flag and generate a boutiques output from a Nipype input spec, code style fixes, some refactoring and improvements --- nipype/utils/nipype2boutiques.py | 169 +++++++++++++++++++------------ 1 file changed, 103 insertions(+), 66 deletions(-) diff --git a/nipype/utils/nipype2boutiques.py b/nipype/utils/nipype2boutiques.py index c5280d2d45..981c8d0cb3 100644 --- a/nipype/utils/nipype2boutiques.py +++ b/nipype/utils/nipype2boutiques.py @@ -9,6 +9,7 @@ # Limitations: # * Optional outputs, i.e. outputs that not always produced, may not be detected. They will, however, still be listed # with a placeholder for the path template (either a value key or the output ID) that should be verified and corrected. +# * Still need to add some fields to the descriptor manually, e.g. url, descriptor-url, path-template-stripped-extensions, etc. import os import sys @@ -77,28 +78,34 @@ def generate_boutiques_descriptor( # Generates tool inputs for name, spec in sorted(interface.inputs.traits(transient=None).items()): - inp = get_boutiques_input(inputs, interface, name, spec, verbose, ignore_inputs=ignore_inputs) - # Handle compound inputs (inputs that can be of multiple types and are mutually exclusive) - if inp is None: + # Skip ignored inputs + if ignore_inputs is not None and name in ignore_inputs: continue - if isinstance(inp, list): - mutex_group_members = [] - tool_desc['command-line'] += inp[0]['value-key'] + " " - for i in inp: - tool_desc['inputs'].append(i) - mutex_group_members.append(i['id']) - if verbose: - print("-> Adding input " + i['name']) - # Put inputs into a mutually exclusive group - tool_desc['groups'].append({'id': inp[0]['id'] + "_group", - 'name': inp[0]['name'] + " group", - 'members': mutex_group_members, - 'mutually-exclusive': True}) + # If spec has a name source, this means it actually represents an output, so create a + # Boutiques output from it + elif spec.name_source and spec.name_template: + tool_desc['output-files'].append(get_boutiques_output_from_inp(inputs, spec, name)) else: - tool_desc['inputs'].append(inp) - tool_desc['command-line'] += inp['value-key'] + " " - if verbose: - print("-> Adding input " + inp['name']) + inp = get_boutiques_input(inputs, interface, name, spec, verbose, ignore_inputs=ignore_inputs) + # Handle compound inputs (inputs that can be of multiple types and are mutually exclusive) + if isinstance(inp, list): + mutex_group_members = [] + tool_desc['command-line'] += inp[0]['value-key'] + " " + for i in inp: + tool_desc['inputs'].append(i) + mutex_group_members.append(i['id']) + if verbose: + print("-> Adding input " + i['name']) + # Put inputs into a mutually exclusive group + tool_desc['groups'].append({'id': inp[0]['id'] + "_group", + 'name': inp[0]['name'] + " group", + 'members': mutex_group_members, + 'mutually-exclusive': True}) + else: + tool_desc['inputs'].append(inp) + tool_desc['command-line'] += inp['value-key'] + " " + if verbose: + print("-> Adding input " + inp['name']) # Generates input groups tool_desc['groups'] += get_boutiques_groups(interface.inputs.traits(transient=None).items()) @@ -148,7 +155,7 @@ def generate_boutiques_descriptor( # Save descriptor to a file if save: - path = save_path if save_path is not None else os.path.join(os.getcwd(), interface_name + '.json') + path = save_path or os.path.join(os.getcwd(), interface_name + '.json') with open(path, 'w') as outfile: json.dump(tool_desc, outfile, indent=4, separators=(',', ': ')) if verbose: @@ -200,15 +207,9 @@ def get_boutiques_input(inputs, interface, input_name, spec, verbose, handler=No Assumes that: * Input names are unique. """ - - # If spec has a name source, means it's an output, so skip it here. - # Also skip any ignored inputs - if spec.name_source or ignore_inputs is not None and input_name in ignore_inputs: - return None - inp = {} - if input_number is not None and input_number != 0: # No need to append a number to the first of a list of compound inputs + if input_number: # No need to append a number to the first of a list of compound inputs inp['id'] = input_name + "_" + str(input_number + 1) else: inp['id'] = input_name @@ -302,20 +303,18 @@ def get_boutiques_input(inputs, interface, input_name, spec, verbose, handler=No if handler_type == "InputMultiObject": inp['type'] = "File" inp['list'] = True + if spec.sep: + inp['list-separator'] = spec.sep inp['value-key'] = "[" + input_name.upper( ) + "]" # assumes that input names are unique - # Add the command line flag specified by argstr - # If no argstr is provided and input type is Flag, create a flag from the name - if spec.argstr: - if "=" in spec.argstr: - inp['command-line-flag'] = spec.argstr.split("=")[0].strip() - inp['command-line-flag-separator'] = "=" - elif spec.argstr.split("%")[0]: - inp['command-line-flag'] = spec.argstr.split("%")[0].strip() - elif inp['type'] == "Flag": - inp['command-line-flag'] = ("--%s" % input_name + " ").strip() + flag, flag_sep = get_command_line_flag(spec, inp['type'] == "Flag", input_name) + + if flag is not None: + inp['command-line-flag'] = flag + if flag_sep is not None: + inp['command-line-flag-separator'] = flag_sep inp['description'] = get_description_from_spec(inputs, input_name, spec) if not (hasattr(spec, "mandatory") and spec.mandatory): @@ -384,42 +383,32 @@ def get_boutiques_output(outputs, name, spec, interface, tool_inputs): output_value = None except AttributeError: output_value = None + except KeyError: + output_value = None # Handle multi-outputs - if isinstance(output_value, list): + if isinstance(output_value, list) or type(spec.handler).__name__ == "OutputMultiObject": output['list'] = True - # Check if all extensions are the same - extensions = [] - for val in output_value: - extensions.append(os.path.splitext(val)[1]) - # If extensions all the same, set path template as wildcard + extension - # Otherwise just use a wildcard - if len(set(extensions)) == 1: - output['path-template'] = "*" + extensions[0] - else: - output['path-template'] = "*" - return output + if output_value: + # Check if all extensions are the same + extensions = [] + for val in output_value: + extensions.append(os.path.splitext(val)[1]) + # If extensions all the same, set path template as wildcard + extension + # Otherwise just use a wildcard + if len(set(extensions)) == 1: + output['path-template'] = "*" + extensions[0] + else: + output['path-template'] = "*" + return output # If an output value is defined, use its relative path, if one exists. - # If no relative path, look for an input with the same name containing a name source - # and name template. Otherwise, put blank string as placeholder and try to fill it on + # Otherwise, put blank string as placeholder and try to fill it on # another iteration. - output['path-template'] = "" - if output_value: output['path-template'] = os.path.relpath(output_value) else: - for inp_name, inp_spec in sorted(interface.inputs.traits(transient=None).items()): - if inp_name == name and inp_spec.name_source and inp_spec.name_template: - if isinstance(inp_spec.name_source, list): - source = inp_spec.name_source[0] - else: - source = inp_spec.name_source - output['path-template'] = inp_spec.name_template.replace("%s", "[" + source.upper() + "]") - output['value-key'] = "[" + name.upper() + "]" - if inp_spec.argstr and inp_spec.argstr.split("%")[0]: - output['command-line-flag'] = inp_spec.argstr.split("%")[0].strip() - break + output['path-template'] = "" return output @@ -535,4 +524,52 @@ def reorder_cmd_line_args(cmd_line, interface, ignore_inputs=None): continue positional_args.append(item[1]) - return interface_name + " " + " ".join(positional_args) + " " + ((last_arg + " ") if last_arg else "") + " ".join(non_positional_args) + return interface_name + " " +\ + ((" ".join(positional_args) + " ") if len(positional_args) > 0 else "") +\ + ((last_arg + " ") if last_arg else "") +\ + " ".join(non_positional_args) + + +def get_command_line_flag(input_spec, is_flag_type=False, input_name=None): + ''' + Generates the command line flag for a given input + ''' + flag, flag_sep = None, None + if input_spec.argstr: + if "=" in input_spec.argstr: + flag = input_spec.argstr.split("=")[0].strip() + flag_sep = "=" + elif input_spec.argstr.split("%")[0]: + flag = input_spec.argstr.split("%")[0].strip() + elif is_flag_type: + flag = ("--%s" % input_name + " ").strip() + return flag, flag_sep + + +def get_boutiques_output_from_inp(inputs, inp_spec, inp_name): + ''' + Takes a Nipype input representing an output file and generates a Boutiques output for it + ''' + output = {} + output['name'] = inp_name.replace('_', ' ').capitalize() + output['id'] = inp_name + output['optional'] = True + output['description'] = get_description_from_spec(inputs, inp_name, inp_spec) + if not (hasattr(inp_spec, "mandatory") and inp_spec.mandatory): + output['optional'] = True + else: + output['optional'] = False + if inp_spec.usedefault: + output['default-value'] = inp_spec.default_value()[1] + if isinstance(inp_spec.name_source, list): + source = inp_spec.name_source[0] + else: + source = inp_spec.name_source + output['path-template'] = inp_spec.name_template.replace("%s", "[" + source.upper() + "]") + output['value-key'] = "[" + inp_name.upper() + "]" + flag, flag_sep = get_command_line_flag(inp_spec) + if flag is not None: + output['command-line-flag'] = flag + if flag_sep is not None: + output['command-line-flag-separator'] = flag_sep + return output From b4b2de077fc94978c30c70355252d8ce9ea68775 Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Wed, 27 Mar 2019 21:20:26 -0400 Subject: [PATCH 21/28] updated nipypecli, fixed code style --- nipype/scripts/cli.py | 61 +++++++----- nipype/utils/nipype2boutiques.py | 166 +++++++++++++++++++------------ 2 files changed, 139 insertions(+), 88 deletions(-) diff --git a/nipype/scripts/cli.py b/nipype/scripts/cli.py index 59d8672cfb..482f359aae 100644 --- a/nipype/scripts/cli.py +++ b/nipype/scripts/cli.py @@ -211,48 +211,55 @@ def convert(): help="JSON file name where the Boutiques descriptor will be " "written.") @click.option( - "-t", - "--ignored-template-inputs", + "-c", + "--container-image", + required=True, type=str, - multiple=True, - help="Interface inputs ignored in path template creations.") + help="Name of the container image where the tool is installed.") @click.option( - "-d", - "--docker-image", + "-p", + "--container-type", + required=True, type=str, - help="Name of the Docker image where the Nipype interface is " - "available.") + help="Type of container image (Docker or Singularity).") @click.option( - "-r", - "--docker-index", + "-x", + "--container-index", type=str, - help="Docker index where the Docker image is stored (e.g. " + help="Optional index where the image is available (e.g. " "http://index.docker.io).") @click.option( - "-n", - "--ignore-template-numbers", - is_flag=True, - flag_value=True, - help="Ignore all numbers in path template creations.") + "-g", + "--ignore-inputs", + type=str, + multiple=True, + help="List of interface inputs to not include in the descriptor.") @click.option( "-v", "--verbose", is_flag=True, flag_value=True, - help="Enable verbose output.") -def boutiques(interface, module, output, ignored_template_inputs, docker_image, - docker_index, ignore_template_numbers, verbose): + help="Print information messages.") +@click.option( + "-a", + "--author", + type=str, + help="Author of the tool (required for publishing).") +@click.option( + "-t", + "--tags", + type=str, + help="JSON string containing tags to include in the descriptor," + "e.g. \"{\"key1\": \"value1\"}\"") +def boutiques(module, interface, container_image, container_type, output, + container_index, verbose, author, ignore_inputs, tags): """Nipype to Boutiques exporter. See Boutiques specification at https://github.com/boutiques/schema. """ from nipype.utils.nipype2boutiques import generate_boutiques_descriptor - # Generates JSON string - json_string = generate_boutiques_descriptor( - module, interface, ignored_template_inputs, docker_image, docker_index, - verbose, ignore_template_numbers) - - # Writes JSON string to file - with open(output, 'w') as f: - f.write(json_string) + # Generates JSON string and saves it to file + generate_boutiques_descriptor( + module, interface, container_image, container_type, container_index, + verbose, True, output, author, ignore_inputs, tags) diff --git a/nipype/utils/nipype2boutiques.py b/nipype/utils/nipype2boutiques.py index 981c8d0cb3..a9500a71a8 100644 --- a/nipype/utils/nipype2boutiques.py +++ b/nipype/utils/nipype2boutiques.py @@ -3,13 +3,18 @@ absolute_import) from builtins import str, open, bytes -# This tool exports a Nipype interface in the Boutiques (https://github.com/boutiques) JSON format. -# Boutiques tools can be imported in CBRAIN (https://github.com/aces/cbrain) among other platforms. +# This tool exports a Nipype interface in the Boutiques +# (https://github.com/boutiques) JSON format. Boutiques tools +# can be imported in CBRAIN (https://github.com/aces/cbrain) +# among other platforms. # # Limitations: -# * Optional outputs, i.e. outputs that not always produced, may not be detected. They will, however, still be listed -# with a placeholder for the path template (either a value key or the output ID) that should be verified and corrected. -# * Still need to add some fields to the descriptor manually, e.g. url, descriptor-url, path-template-stripped-extensions, etc. +# * Optional outputs, i.e. outputs that not always produced, may not be +# detected. They will, however, still be listed with a placeholder for +# the path template (either a value key or the output ID) that should +# be verified and corrected. +# * Still need to add some fields to the descriptor manually, e.g. url, +# descriptor-url, path-template-stripped-extensions, etc. import os import sys @@ -20,10 +25,12 @@ def generate_boutiques_descriptor( - module, interface_name, container_image, container_type, container_index=None, - verbose=False, save=False, save_path=None, author=None, ignore_inputs=None, tags=None): + module, interface_name, container_image, container_type, + container_index=None, verbose=False, save=False, save_path=None, + author=None, ignore_inputs=None, tags=None): ''' - Returns a JSON string containing a JSON Boutiques description of a Nipype interface. + Returns a JSON string containing a JSON Boutiques description of a + Nipype interface. Arguments: * module: module where the Nipype interface is declared. * interface_name: name of Nipype interface. @@ -32,11 +39,13 @@ def generate_boutiques_descriptor( * container_index: optional index where the image is available * verbose: print information messages * save: True if you want to save descriptor to a file - * save_path: file path for the saved descriptor (defaults to name of the interface in current directory) + * save_path: file path for the saved descriptor (defaults to name of the + interface in current directory) * author: author of the tool (required for publishing) * ignore_inputs: list of interface inputs to not include in the descriptor - * tags: JSON object containing tags to include in the descriptor, e.g. "{/"key1/": /"value1/"}" - (note: the tags 'domain:neuroinformatics' and 'interface-type:nipype' are included by default) + * tags: JSON object containing tags to include in the descriptor, + e.g. "{\"key1\": \"value1\"}" (note: the tags 'domain:neuroinformatics' + and 'interface-type:nipype' are included by default) ''' if not module: @@ -62,12 +71,15 @@ def generate_boutiques_descriptor( tool_desc['author'] = "Nipype (interface)" if author is not None: tool_desc['author'] = tool_desc['author'] + ", " + author + " (tool)" - tool_desc[ - 'description'] = interface_name + ", as implemented in Nipype (module: " + module_name + ", interface: " + interface_name + ")." + tool_desc['description'] = (interface_name + + ", as implemented in Nipype (module: " + + module_name + ", interface: " + + interface_name + ").") tool_desc['inputs'] = [] tool_desc['output-files'] = [] tool_desc['groups'] = [] - tool_desc['tool-version'] = interface.version if interface.version is not None else "1.0.0" + tool_desc['tool-version'] = interface.version \ + if interface.version is not None else "1.0.0" tool_desc['schema-version'] = '0.5' if container_image: tool_desc['container-image'] = {} @@ -81,13 +93,15 @@ def generate_boutiques_descriptor( # Skip ignored inputs if ignore_inputs is not None and name in ignore_inputs: continue - # If spec has a name source, this means it actually represents an output, so create a - # Boutiques output from it + # If spec has a name source, this means it actually represents an + # output, so create a Boutiques output from it elif spec.name_source and spec.name_template: - tool_desc['output-files'].append(get_boutiques_output_from_inp(inputs, spec, name)) + tool_desc['output-files']\ + .append(get_boutiques_output_from_inp(inputs, spec, name)) else: - inp = get_boutiques_input(inputs, interface, name, spec, verbose, ignore_inputs=ignore_inputs) - # Handle compound inputs (inputs that can be of multiple types and are mutually exclusive) + inp = get_boutiques_input(inputs, interface, name, spec, verbose) + # Handle compound inputs (inputs that can be of multiple types + # and are mutually exclusive) if isinstance(inp, list): mutex_group_members = [] tool_desc['command-line'] += inp[0]['value-key'] + " " @@ -108,7 +122,8 @@ def generate_boutiques_descriptor( print("-> Adding input " + inp['name']) # Generates input groups - tool_desc['groups'] += get_boutiques_groups(interface.inputs.traits(transient=None).items()) + tool_desc['groups'] +=\ + get_boutiques_groups(interface.inputs.traits(transient=None).items()) if len(tool_desc['groups']) == 0: del tool_desc['groups'] @@ -127,7 +142,8 @@ def generate_boutiques_descriptor( # Fill in all missing output paths for output in tool_desc['output-files']: if output['path-template'] == "": - fill_in_missing_output_path(output, output['name'], tool_desc['inputs']) + fill_in_missing_output_path(output, output['name'], + tool_desc['inputs']) # Add tags desc_tags = { @@ -148,7 +164,8 @@ def generate_boutiques_descriptor( tool_desc['tags'] = desc_tags # Check for positional arguments and reorder command line args if necessary - tool_desc['command-line'] = reorder_cmd_line_args(tool_desc['command-line'], interface, ignore_inputs) + tool_desc['command-line'] = reorder_cmd_line_args( + tool_desc['command-line'], interface, ignore_inputs) # Remove the extra space at the end of the command line tool_desc['command-line'] = tool_desc['command-line'].strip() @@ -161,16 +178,19 @@ def generate_boutiques_descriptor( if verbose: print("-> Descriptor saved to file " + outfile.name) - print("NOTE: Descriptors produced by this script may not entirely conform to the Nipype interface " - "specs. Please check that the descriptor is correct before using it.") + print("NOTE: Descriptors produced by this script may not entirely conform " + "to the Nipype interface specs. Please check that the descriptor is " + "correct before using it.") return json.dumps(tool_desc, indent=4, separators=(',', ': ')) def generate_tool_outputs(outputs, interface, tool_desc, verbose, first_run): for name, spec in sorted(outputs.traits(transient=None).items()): - output = get_boutiques_output(outputs, name, spec, interface, tool_desc['inputs']) - # If this is the first time we are generating outputs, add the full output to the descriptor. - # Otherwise, find the existing output and update its path template if it's still undefined. + output = get_boutiques_output(outputs, name, spec, interface, + tool_desc['inputs']) + # If this is the first time we are generating outputs, add the full + # output to the descriptor. Otherwise, find the existing output and + # update its path template if it's still undefined. if first_run: tool_desc['output-files'].append(output) if output.get('value-key'): @@ -179,20 +199,23 @@ def generate_tool_outputs(outputs, interface, tool_desc, verbose, first_run): print("-> Adding output " + output['name']) else: for existing_output in tool_desc['output-files']: - if output['id'] == existing_output['id'] and existing_output['path-template'] == "": + if (output['id'] == existing_output['id'] and + existing_output['path-template'] == ""): existing_output['path-template'] = output['path-template'] break - if output.get('value-key') and output['value-key'] not in tool_desc['command-line']: + if (output.get('value-key') and + output['value-key'] not in tool_desc['command-line']): tool_desc['command-line'] += output['value-key'] + " " if len(tool_desc['output-files']) == 0: raise Exception("Tool has no output.") -def get_boutiques_input(inputs, interface, input_name, spec, verbose, handler=None, - input_number=None, ignore_inputs=None): +def get_boutiques_input(inputs, interface, input_name, spec, verbose, + handler=None, input_number=None): """ - Returns a dictionary containing the Boutiques input corresponding to a Nipype intput. + Returns a dictionary containing the Boutiques input corresponding + to a Nipype input. Args: * inputs: inputs of the Nipype interface. @@ -200,16 +223,18 @@ def get_boutiques_input(inputs, interface, input_name, spec, verbose, handler=No * input_name: name of the Nipype input. * spec: Nipype input spec. * verbose: print information messages. - * handler: used when handling compound inputs, which don't have their own input spec - * input_number: used when handling compound inputs to assign each a unique ID - * ignore_inputs: list of interface inputs to not include in the descriptor + * handler: used when handling compound inputs, which don't have their + own input spec + * input_number: used when handling compound inputs to assign each a + unique ID Assumes that: * Input names are unique. """ inp = {} - if input_number: # No need to append a number to the first of a list of compound inputs + # No need to append a number to the first of a list of compound inputs + if input_number: inp['id'] = input_name + "_" + str(input_number + 1) else: inp['id'] = input_name @@ -243,7 +268,9 @@ def get_boutiques_input(inputs, interface, input_name, spec, verbose, handler=No elif handler_type == "Float": inp['type'] = "Number" elif handler_type == "Bool": - if spec.argstr and len(spec.argstr.split("=")) > 1 and (spec.argstr.split("=")[1] == '0' or spec.argstr.split("=")[1] == '1'): + if (spec.argstr and len(spec.argstr.split("=")) > 1 and + (spec.argstr.split("=")[1] == '0' + or spec.argstr.split("=")[1] == '1')): inp['type'] = "Number" inp['integer'] = True inp['minimum'] = 0 @@ -309,7 +336,8 @@ def get_boutiques_input(inputs, interface, input_name, spec, verbose, handler=No inp['value-key'] = "[" + input_name.upper( ) + "]" # assumes that input names are unique - flag, flag_sep = get_command_line_flag(spec, inp['type'] == "Flag", input_name) + flag, flag_sep = get_command_line_flag(spec, inp['type'] == "Flag", + input_name) if flag is not None: inp['command-line-flag'] = flag @@ -342,14 +370,16 @@ def get_boutiques_input(inputs, interface, input_name, spec, verbose, handler=No def get_boutiques_output(outputs, name, spec, interface, tool_inputs): """ - Returns a dictionary containing the Boutiques output corresponding to a Nipype output. + Returns a dictionary containing the Boutiques output corresponding + to a Nipype output. Args: * outputs: outputs of the Nipype interface. * name: name of the Nipype output. * spec: Nipype output spec. * interface: Nipype interface. - * tool_inputs: list of tool inputs (as produced by method get_boutiques_input). + * tool_inputs: list of tool inputs (as produced by method + get_boutiques_input). Assumes that: * Output names are unique. @@ -370,8 +400,10 @@ def get_boutiques_output(outputs, name, spec, interface, tool_inputs): output['id'] = name if unique_id else name + '_outfile' output['path-template'] = "" - output[ - 'optional'] = True # no real way to determine if an output is always produced, regardless of the input values. + + # No real way to determine if an output is always + # produced, regardless of the input values. + output['optional'] = True output['description'] = get_description_from_spec(outputs, name, spec) @@ -387,15 +419,16 @@ def get_boutiques_output(outputs, name, spec, interface, tool_inputs): output_value = None # Handle multi-outputs - if isinstance(output_value, list) or type(spec.handler).__name__ == "OutputMultiObject": + if (isinstance(output_value, list) or + type(spec.handler).__name__ == "OutputMultiObject"): output['list'] = True if output_value: # Check if all extensions are the same extensions = [] for val in output_value: extensions.append(os.path.splitext(val)[1]) - # If extensions all the same, set path template as wildcard + extension - # Otherwise just use a wildcard + # If extensions all the same, set path template as + # wildcard + extension. Otherwise just use a wildcard if len(set(extensions)) == 1: output['path-template'] = "*" + extensions[0] else: @@ -415,7 +448,8 @@ def get_boutiques_output(outputs, name, spec, interface, tool_inputs): def get_boutiques_groups(input_traits): """ - Returns a list of dictionaries containing Boutiques groups for the mutually exclusive and all-or-none Nipype inputs. + Returns a list of dictionaries containing Boutiques groups for the mutually + exclusive and all-or-none Nipype inputs. """ desc_groups = [] all_or_none_input_sets = [] @@ -434,14 +468,18 @@ def get_boutiques_groups(input_traits): # Create a dictionary for each one for i, inp_set in enumerate(all_or_none_input_sets, 1): - desc_groups.append({'id': "all_or_none_group" + ("_" + str(i) if i != 1 else ""), - 'name': "All or none group" + (" " + str(i) if i != 1 else ""), + desc_groups.append({'id': "all_or_none_group" + + ("_" + str(i) if i != 1 else ""), + 'name': "All or none group" + + (" " + str(i) if i != 1 else ""), 'members': list(inp_set), 'all-or-none': True}) for i, inp_set in enumerate(mutex_input_sets, 1): - desc_groups.append({'id': "mutex_group" + ("_" + str(i) if i != 1 else ""), - 'name': "Mutex group" + (" " + str(i) if i != 1 else ""), + desc_groups.append({'id': "mutex_group" + + ("_" + str(i) if i != 1 else ""), + 'name': "Mutex group" + + (" " + str(i) if i != 1 else ""), 'members': list(inp_set), 'mutually-exclusive': True}) @@ -485,9 +523,10 @@ def fill_in_missing_output_path(output, output_name, tool_inputs): def generate_custom_inputs(desc_inputs): ''' - Generates a bunch of custom input dictionaries in order to generate as many outputs as possible - (to get their path templates) - Currently only works with flag inputs and inputs with defined value choices. + Generates a bunch of custom input dictionaries in order to generate + as many outputs as possible (to get their path templates). + Currently only works with flag inputs and inputs with defined value + choices. ''' custom_input_dicts = [] for desc_input in desc_inputs: @@ -501,7 +540,8 @@ def generate_custom_inputs(desc_inputs): def reorder_cmd_line_args(cmd_line, interface, ignore_inputs=None): ''' - Generates a new command line with the positional arguments in the correct order + Generates a new command line with the positional arguments in the + correct order ''' interface_name = cmd_line.split()[0] positional_arg_dict = {} @@ -524,10 +564,11 @@ def reorder_cmd_line_args(cmd_line, interface, ignore_inputs=None): continue positional_args.append(item[1]) - return interface_name + " " +\ - ((" ".join(positional_args) + " ") if len(positional_args) > 0 else "") +\ - ((last_arg + " ") if last_arg else "") +\ - " ".join(non_positional_args) + return (interface_name + " " + + ((" ".join(positional_args) + " ") + if len(positional_args) > 0 else "") + + ((last_arg + " ") if last_arg else "") + + " ".join(non_positional_args)) def get_command_line_flag(input_spec, is_flag_type=False, input_name=None): @@ -548,13 +589,15 @@ def get_command_line_flag(input_spec, is_flag_type=False, input_name=None): def get_boutiques_output_from_inp(inputs, inp_spec, inp_name): ''' - Takes a Nipype input representing an output file and generates a Boutiques output for it + Takes a Nipype input representing an output file and generates a + Boutiques output for it ''' output = {} output['name'] = inp_name.replace('_', ' ').capitalize() output['id'] = inp_name output['optional'] = True - output['description'] = get_description_from_spec(inputs, inp_name, inp_spec) + output['description'] = get_description_from_spec(inputs, inp_name, + inp_spec) if not (hasattr(inp_spec, "mandatory") and inp_spec.mandatory): output['optional'] = True else: @@ -565,7 +608,8 @@ def get_boutiques_output_from_inp(inputs, inp_spec, inp_name): source = inp_spec.name_source[0] else: source = inp_spec.name_source - output['path-template'] = inp_spec.name_template.replace("%s", "[" + source.upper() + "]") + output['path-template'] = inp_spec.name_template.replace( + "%s", "[" + source.upper() + "]") output['value-key'] = "[" + inp_name.upper() + "]" flag, flag_sep = get_command_line_flag(inp_spec) if flag is not None: From ebe7da702d84a405901a2aabb11e5873207b9622 Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Thu, 28 Mar 2019 20:47:03 -0400 Subject: [PATCH 22/28] updated nipype2boutiques test --- nipype/utils/nipype2boutiques_example.json | 549 ++++++++++++++++++++ nipype/utils/tests/test_nipype2boutiques.py | 42 +- 2 files changed, 582 insertions(+), 9 deletions(-) create mode 100644 nipype/utils/nipype2boutiques_example.json diff --git a/nipype/utils/nipype2boutiques_example.json b/nipype/utils/nipype2boutiques_example.json new file mode 100644 index 0000000000..45359f49ed --- /dev/null +++ b/nipype/utils/nipype2boutiques_example.json @@ -0,0 +1,549 @@ +{ + "name": "FLIRT", + "command-line": "FLIRT [IN_FILE] [REFERENCE] [OUT_FILE] [OUT_MATRIX_FILE] [ANGLE_REP] [APPLY_ISOXFM] [APPLY_XFM] [BBRSLOPE] [BBRTYPE] [BGVALUE] [BINS] [COARSE_SEARCH] [COST] [COST_FUNC] [DATATYPE] [DISPLAY_INIT] [DOF] [ECHOSPACING] [FIELDMAP] [FIELDMAPMASK] [FINE_SEARCH] [FORCE_SCALING] [IN_MATRIX_FILE] [IN_WEIGHT] [INTERP] [MIN_SAMPLING] [NO_CLAMP] [NO_RESAMPLE] [NO_RESAMPLE_BLUR] [NO_SEARCH] [OUT_LOG] [PADDING_SIZE] [PEDIR] [REF_WEIGHT] [RIGID2D] [SAVE_LOG] [SCHEDULE] [SEARCHR_X] [SEARCHR_Y] [SEARCHR_Z] [SINC_WIDTH] [SINC_WINDOW] [USES_QFORM] [VERBOSE] [WM_SEG] [WMCOORDS] [WMNORMS]", + "author": "Nipype (interface), Oxford Centre for Functional MRI of the Brain (FMRIB) (tool)", + "description": "FLIRT, as implemented in Nipype (module: nipype.interfaces.fsl, interface: FLIRT).", + "inputs": [ + { + "id": "angle_rep", + "name": "Angle rep", + "type": "String", + "value-key": "[ANGLE_REP]", + "command-line-flag": "-anglerep", + "description": "'quaternion' or 'euler'. Representation of rotation angles.", + "optional": true, + "value-choices": [ + "quaternion", + "euler" + ] + }, + { + "id": "apply_isoxfm", + "name": "Apply isoxfm", + "type": "Number", + "value-key": "[APPLY_ISOXFM]", + "command-line-flag": "-applyisoxfm", + "description": "A float. As applyxfm but forces isotropic resampling.", + "optional": true + }, + { + "id": "apply_xfm", + "name": "Apply xfm", + "type": "Flag", + "value-key": "[APPLY_XFM]", + "command-line-flag": "-applyxfm", + "description": "A boolean. Apply transformation supplied by in_matrix_file or uses_qform to use the affine matrix stored in the reference header.", + "optional": true + }, + { + "id": "bbrslope", + "name": "Bbrslope", + "type": "Number", + "value-key": "[BBRSLOPE]", + "command-line-flag": "-bbrslope", + "description": "A float. Value of bbr slope.", + "optional": true + }, + { + "id": "bbrtype", + "name": "Bbrtype", + "type": "String", + "value-key": "[BBRTYPE]", + "command-line-flag": "-bbrtype", + "description": "'signed' or 'global_abs' or 'local_abs'. Type of bbr cost function: signed [default], global_abs, local_abs.", + "optional": true, + "value-choices": [ + "signed", + "global_abs", + "local_abs" + ] + }, + { + "id": "bgvalue", + "name": "Bgvalue", + "type": "Number", + "value-key": "[BGVALUE]", + "command-line-flag": "-setbackground", + "description": "A float. Use specified background value for points outside fov.", + "optional": true + }, + { + "id": "bins", + "name": "Bins", + "type": "Number", + "integer": true, + "value-key": "[BINS]", + "command-line-flag": "-bins", + "description": "An integer (int or long). Number of histogram bins.", + "optional": true + }, + { + "id": "coarse_search", + "name": "Coarse search", + "type": "Number", + "integer": true, + "value-key": "[COARSE_SEARCH]", + "command-line-flag": "-coarsesearch", + "description": "An integer (int or long). Coarse search delta angle.", + "optional": true + }, + { + "id": "cost", + "name": "Cost", + "type": "String", + "value-key": "[COST]", + "command-line-flag": "-cost", + "description": "'mutualinfo' or 'corratio' or 'normcorr' or 'normmi' or 'leastsq' or 'labeldiff' or 'bbr'. Cost function.", + "optional": true, + "value-choices": [ + "mutualinfo", + "corratio", + "normcorr", + "normmi", + "leastsq", + "labeldiff", + "bbr" + ] + }, + { + "id": "cost_func", + "name": "Cost func", + "type": "String", + "value-key": "[COST_FUNC]", + "command-line-flag": "-searchcost", + "description": "'mutualinfo' or 'corratio' or 'normcorr' or 'normmi' or 'leastsq' or 'labeldiff' or 'bbr'. Cost function.", + "optional": true, + "value-choices": [ + "mutualinfo", + "corratio", + "normcorr", + "normmi", + "leastsq", + "labeldiff", + "bbr" + ] + }, + { + "id": "datatype", + "name": "Datatype", + "type": "String", + "value-key": "[DATATYPE]", + "command-line-flag": "-datatype", + "description": "'char' or 'short' or 'int' or 'float' or 'double'. Force output data type.", + "optional": true, + "value-choices": [ + "char", + "short", + "int", + "float", + "double" + ] + }, + { + "id": "display_init", + "name": "Display init", + "type": "Flag", + "value-key": "[DISPLAY_INIT]", + "command-line-flag": "-displayinit", + "description": "A boolean. Display initial matrix.", + "optional": true + }, + { + "id": "dof", + "name": "Dof", + "type": "Number", + "integer": true, + "value-key": "[DOF]", + "command-line-flag": "-dof", + "description": "An integer (int or long). Number of transform degrees of freedom.", + "optional": true + }, + { + "id": "echospacing", + "name": "Echospacing", + "type": "Number", + "value-key": "[ECHOSPACING]", + "command-line-flag": "-echospacing", + "description": "A float. Value of epi echo spacing - units of seconds.", + "optional": true + }, + { + "id": "fieldmap", + "name": "Fieldmap", + "type": "File", + "value-key": "[FIELDMAP]", + "command-line-flag": "-fieldmap", + "description": "A file name. Fieldmap image in rads/s - must be already registered to the reference image.", + "optional": true + }, + { + "id": "fieldmapmask", + "name": "Fieldmapmask", + "type": "File", + "value-key": "[FIELDMAPMASK]", + "command-line-flag": "-fieldmapmask", + "description": "A file name. Mask for fieldmap image.", + "optional": true + }, + { + "id": "fine_search", + "name": "Fine search", + "type": "Number", + "integer": true, + "value-key": "[FINE_SEARCH]", + "command-line-flag": "-finesearch", + "description": "An integer (int or long). Fine search delta angle.", + "optional": true + }, + { + "id": "force_scaling", + "name": "Force scaling", + "type": "Flag", + "value-key": "[FORCE_SCALING]", + "command-line-flag": "-forcescaling", + "description": "A boolean. Force rescaling even for low-res images.", + "optional": true + }, + { + "id": "in_file", + "name": "In file", + "type": "File", + "value-key": "[IN_FILE]", + "command-line-flag": "-in", + "description": "An existing file name. Input file.", + "optional": false + }, + { + "id": "in_matrix_file", + "name": "In matrix file", + "type": "File", + "value-key": "[IN_MATRIX_FILE]", + "command-line-flag": "-init", + "description": "A file name. Input 4x4 affine matrix.", + "optional": true + }, + { + "id": "in_weight", + "name": "In weight", + "type": "File", + "value-key": "[IN_WEIGHT]", + "command-line-flag": "-inweight", + "description": "An existing file name. File for input weighting volume.", + "optional": true + }, + { + "id": "interp", + "name": "Interp", + "type": "String", + "value-key": "[INTERP]", + "command-line-flag": "-interp", + "description": "'trilinear' or 'nearestneighbour' or 'sinc' or 'spline'. Final interpolation method used in reslicing.", + "optional": true, + "value-choices": [ + "trilinear", + "nearestneighbour", + "sinc", + "spline" + ] + }, + { + "id": "min_sampling", + "name": "Min sampling", + "type": "Number", + "value-key": "[MIN_SAMPLING]", + "command-line-flag": "-minsampling", + "description": "A float. Set minimum voxel dimension for sampling.", + "optional": true + }, + { + "id": "no_clamp", + "name": "No clamp", + "type": "Flag", + "value-key": "[NO_CLAMP]", + "command-line-flag": "-noclamp", + "description": "A boolean. Do not use intensity clamping.", + "optional": true + }, + { + "id": "no_resample", + "name": "No resample", + "type": "Flag", + "value-key": "[NO_RESAMPLE]", + "command-line-flag": "-noresample", + "description": "A boolean. Do not change input sampling.", + "optional": true + }, + { + "id": "no_resample_blur", + "name": "No resample blur", + "type": "Flag", + "value-key": "[NO_RESAMPLE_BLUR]", + "command-line-flag": "-noresampblur", + "description": "A boolean. Do not use blurring on downsampling.", + "optional": true + }, + { + "id": "no_search", + "name": "No search", + "type": "Flag", + "value-key": "[NO_SEARCH]", + "command-line-flag": "-nosearch", + "description": "A boolean. Set all angular searches to ranges 0 to 0.", + "optional": true + }, + { + "id": "padding_size", + "name": "Padding size", + "type": "Number", + "integer": true, + "value-key": "[PADDING_SIZE]", + "command-line-flag": "-paddingsize", + "description": "An integer (int or long). For applyxfm: interpolates outside image by size.", + "optional": true + }, + { + "id": "pedir", + "name": "Pedir", + "type": "Number", + "integer": true, + "value-key": "[PEDIR]", + "command-line-flag": "-pedir", + "description": "An integer (int or long). Phase encode direction of epi - 1/2/3=x/y/z & -1/-2/-3=-x/-y/-z.", + "optional": true + }, + { + "id": "ref_weight", + "name": "Ref weight", + "type": "File", + "value-key": "[REF_WEIGHT]", + "command-line-flag": "-refweight", + "description": "An existing file name. File for reference weighting volume.", + "optional": true + }, + { + "id": "reference", + "name": "Reference", + "type": "File", + "value-key": "[REFERENCE]", + "command-line-flag": "-ref", + "description": "An existing file name. Reference file.", + "optional": false + }, + { + "id": "rigid2D", + "name": "Rigid2d", + "type": "Flag", + "value-key": "[RIGID2D]", + "command-line-flag": "-2D", + "description": "A boolean. Use 2d rigid body mode - ignores dof.", + "optional": true + }, + { + "id": "save_log", + "name": "Save log", + "type": "Flag", + "value-key": "[SAVE_LOG]", + "command-line-flag": "--save_log", + "description": "A boolean. Save to log file.", + "optional": true + }, + { + "id": "schedule", + "name": "Schedule", + "type": "File", + "value-key": "[SCHEDULE]", + "command-line-flag": "-schedule", + "description": "An existing file name. Replaces default schedule.", + "optional": true + }, + { + "id": "searchr_x", + "name": "Searchr x", + "type": "Number", + "list": true, + "integer": true, + "min-list-entries": 2, + "max-list-entries": 2, + "value-key": "[SEARCHR_X]", + "command-line-flag": "-searchrx", + "description": "A list of from 2 to 2 items which are an integer (int or long). Search angles along x-axis, in degrees.", + "optional": true + }, + { + "id": "searchr_y", + "name": "Searchr y", + "type": "Number", + "list": true, + "integer": true, + "min-list-entries": 2, + "max-list-entries": 2, + "value-key": "[SEARCHR_Y]", + "command-line-flag": "-searchry", + "description": "A list of from 2 to 2 items which are an integer (int or long). Search angles along y-axis, in degrees.", + "optional": true + }, + { + "id": "searchr_z", + "name": "Searchr z", + "type": "Number", + "list": true, + "integer": true, + "min-list-entries": 2, + "max-list-entries": 2, + "value-key": "[SEARCHR_Z]", + "command-line-flag": "-searchrz", + "description": "A list of from 2 to 2 items which are an integer (int or long). Search angles along z-axis, in degrees.", + "optional": true + }, + { + "id": "sinc_width", + "name": "Sinc width", + "type": "Number", + "integer": true, + "value-key": "[SINC_WIDTH]", + "command-line-flag": "-sincwidth", + "description": "An integer (int or long). Full-width in voxels.", + "optional": true + }, + { + "id": "sinc_window", + "name": "Sinc window", + "type": "String", + "value-key": "[SINC_WINDOW]", + "command-line-flag": "-sincwindow", + "description": "'rectangular' or 'hanning' or 'blackman'. Sinc window.", + "optional": true, + "value-choices": [ + "rectangular", + "hanning", + "blackman" + ] + }, + { + "id": "uses_qform", + "name": "Uses qform", + "type": "Flag", + "value-key": "[USES_QFORM]", + "command-line-flag": "-usesqform", + "description": "A boolean. Initialize using sform or qform.", + "optional": true + }, + { + "id": "verbose", + "name": "Verbose", + "type": "Number", + "integer": true, + "value-key": "[VERBOSE]", + "command-line-flag": "-verbose", + "description": "An integer (int or long). Verbose mode, 0 is least.", + "optional": true + }, + { + "id": "wm_seg", + "name": "Wm seg", + "type": "File", + "value-key": "[WM_SEG]", + "command-line-flag": "-wmseg", + "description": "A file name. White matter segmentation volume needed by bbr cost function.", + "optional": true + }, + { + "id": "wmcoords", + "name": "Wmcoords", + "type": "File", + "value-key": "[WMCOORDS]", + "command-line-flag": "-wmcoords", + "description": "A file name. White matter boundary coordinates for bbr cost function.", + "optional": true + }, + { + "id": "wmnorms", + "name": "Wmnorms", + "type": "File", + "value-key": "[WMNORMS]", + "command-line-flag": "-wmnorms", + "description": "A file name. White matter boundary normals for bbr cost function.", + "optional": true + } + ], + "output-files": [ + { + "name": "Out file", + "id": "out_file", + "optional": true, + "description": "A file name. Registered output file.", + "path-template": "[IN_FILE]_flirt", + "value-key": "[OUT_FILE]", + "command-line-flag": "-out" + }, + { + "name": "Out log", + "id": "out_log", + "optional": true, + "description": "A file name. Output log.", + "path-template": "[IN_FILE]_flirt.log", + "value-key": "[OUT_LOG]" + }, + { + "name": "Out matrix file", + "id": "out_matrix_file", + "optional": true, + "description": "A file name. Output affine matrix in 4x4 asciii format.", + "path-template": "[IN_FILE]_flirt.mat", + "value-key": "[OUT_MATRIX_FILE]", + "command-line-flag": "-omat" + }, + { + "name": "Out file", + "id": "out_file", + "path-template": "out_file", + "optional": true, + "description": "An existing file name. Path/name of registered file (if generated)." + }, + { + "name": "Out log", + "id": "out_log", + "path-template": "out_log", + "optional": true, + "description": "A file name. Path/name of output log (if generated)." + }, + { + "name": "Out matrix file", + "id": "out_matrix_file", + "path-template": "out_matrix_file", + "optional": true, + "description": "An existing file name. Path/name of calculated affine transform (if generated)." + } + ], + "groups": [ + { + "id": "all_or_none_group", + "name": "All or none group", + "members": [ + "save_log", + "out_log" + ], + "all-or-none": true + }, + { + "id": "mutex_group", + "name": "Mutex group", + "members": [ + "apply_isoxfm", + "apply_xfm" + ], + "mutually-exclusive": true + } + ], + "tool-version": "1.0.0", + "schema-version": "0.5", + "container-image": { + "image": "mcin/docker-fsl:latest", + "type": "docker", + "index": "index.docker.io" + }, + "tags": { + "domain": "neuroinformatics", + "source": "nipype-interface" + } +} \ No newline at end of file diff --git a/nipype/utils/tests/test_nipype2boutiques.py b/nipype/utils/tests/test_nipype2boutiques.py index 926de0f2cd..f5c6c6494a 100644 --- a/nipype/utils/tests/test_nipype2boutiques.py +++ b/nipype/utils/tests/test_nipype2boutiques.py @@ -2,16 +2,40 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: from future import standard_library -standard_library.install_aliases() - from ..nipype2boutiques import generate_boutiques_descriptor +import json +standard_library.install_aliases() def test_generate(): - generate_boutiques_descriptor(module='nipype.interfaces.ants.registration', - interface_name='ANTS', - container_image=None, - container_index=None, - container_type=None, - verbose=False, - save=False) + ignored_inputs = [ + "args", + "environ", + "output_type" + ] + desc = generate_boutiques_descriptor(module='nipype.interfaces.fsl', + interface_name='FLIRT', + container_image=('mcin/' + 'docker-fsl:latest'), + container_index='index.docker.io', + container_type='docker', + verbose=False, + save=False, + ignore_inputs=ignored_inputs, + author=("Oxford Centre for Functional" + " MRI of the Brain (FMRIB)")) + + with open('nipype/utils/nipype2boutiques_example.json', 'r', + encoding='utf-8') as desc_file: + assert ordered(json.loads(desc)) == ordered(json.load(desc_file)) + + +# Recursively sorts all items in a JSON object +# Used when comparing two JSON objects whose ordering may differ +def ordered(obj): + if isinstance(obj, dict): + return sorted((k, ordered(v)) for k, v in obj.items()) + if isinstance(obj, list): + return sorted(ordered(x) for x in obj) + else: + return obj From 2ff50eda8cf09914867ad72212e064735216600e Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Thu, 28 Mar 2019 21:55:58 -0400 Subject: [PATCH 23/28] fix test file path --- nipype/utils/tests/test_nipype2boutiques.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/utils/tests/test_nipype2boutiques.py b/nipype/utils/tests/test_nipype2boutiques.py index f5c6c6494a..520f3c7862 100644 --- a/nipype/utils/tests/test_nipype2boutiques.py +++ b/nipype/utils/tests/test_nipype2boutiques.py @@ -25,7 +25,7 @@ def test_generate(): author=("Oxford Centre for Functional" " MRI of the Brain (FMRIB)")) - with open('nipype/utils/nipype2boutiques_example.json', 'r', + with open('utils/nipype2boutiques_example.json', 'r', encoding='utf-8') as desc_file: assert ordered(json.loads(desc)) == ordered(json.load(desc_file)) From c828feca0efe3108736e0869da33689082897b4c Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Fri, 29 Mar 2019 16:46:24 -0400 Subject: [PATCH 24/28] fixed test --- nipype/{utils => testing/data}/nipype2boutiques_example.json | 0 nipype/utils/tests/test_nipype2boutiques.py | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename nipype/{utils => testing/data}/nipype2boutiques_example.json (100%) diff --git a/nipype/utils/nipype2boutiques_example.json b/nipype/testing/data/nipype2boutiques_example.json similarity index 100% rename from nipype/utils/nipype2boutiques_example.json rename to nipype/testing/data/nipype2boutiques_example.json diff --git a/nipype/utils/tests/test_nipype2boutiques.py b/nipype/utils/tests/test_nipype2boutiques.py index 520f3c7862..f3ad944650 100644 --- a/nipype/utils/tests/test_nipype2boutiques.py +++ b/nipype/utils/tests/test_nipype2boutiques.py @@ -3,6 +3,7 @@ # vi: set ft=python sts=4 ts=4 sw=4 et: from future import standard_library from ..nipype2boutiques import generate_boutiques_descriptor +from nipype.testing import example_data import json standard_library.install_aliases() @@ -25,8 +26,7 @@ def test_generate(): author=("Oxford Centre for Functional" " MRI of the Brain (FMRIB)")) - with open('utils/nipype2boutiques_example.json', 'r', - encoding='utf-8') as desc_file: + with open(example_data('nipype2boutiques_example.json'), 'r') as desc_file: assert ordered(json.loads(desc)) == ordered(json.load(desc_file)) From 7e96311e26f8589e980576a1b6f13fcd9796a51d Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Tue, 2 Apr 2019 10:33:05 -0400 Subject: [PATCH 25/28] changed test to only check a subset of descriptor fields --- nipype/utils/tests/test_nipype2boutiques.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/nipype/utils/tests/test_nipype2boutiques.py b/nipype/utils/tests/test_nipype2boutiques.py index f3ad944650..5bba16be6f 100644 --- a/nipype/utils/tests/test_nipype2boutiques.py +++ b/nipype/utils/tests/test_nipype2boutiques.py @@ -27,7 +27,21 @@ def test_generate(): " MRI of the Brain (FMRIB)")) with open(example_data('nipype2boutiques_example.json'), 'r') as desc_file: - assert ordered(json.loads(desc)) == ordered(json.load(desc_file)) + # Make sure that output descriptor matches the expected descriptor. + output_desc = json.loads(desc) + expected_desc = json.load(desc_file) + assert (output_desc.get('name') == + expected_desc.get('name')) + assert (output_desc.get('author') == + expected_desc.get('author')) + assert (output_desc.get('command-line') == + expected_desc.get('command-line')) + assert (output_desc.get('description') == + expected_desc.get('description')) + assert (ordered(output_desc.get('inputs')) == + ordered(expected_desc.get('inputs'))) + assert (ordered(output_desc.get('container-image')) == + ordered(expected_desc.get('container-image'))) # Recursively sorts all items in a JSON object From 0eae216930657d07f80ee02b0a0e1dd93e93cd00 Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Thu, 4 Apr 2019 14:53:37 -0400 Subject: [PATCH 26/28] changed test again --- nipype/utils/tests/test_nipype2boutiques.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/nipype/utils/tests/test_nipype2boutiques.py b/nipype/utils/tests/test_nipype2boutiques.py index 5bba16be6f..19735df6b5 100644 --- a/nipype/utils/tests/test_nipype2boutiques.py +++ b/nipype/utils/tests/test_nipype2boutiques.py @@ -38,18 +38,9 @@ def test_generate(): expected_desc.get('command-line')) assert (output_desc.get('description') == expected_desc.get('description')) - assert (ordered(output_desc.get('inputs')) == - ordered(expected_desc.get('inputs'))) - assert (ordered(output_desc.get('container-image')) == - ordered(expected_desc.get('container-image'))) - - -# Recursively sorts all items in a JSON object -# Used when comparing two JSON objects whose ordering may differ -def ordered(obj): - if isinstance(obj, dict): - return sorted((k, ordered(v)) for k, v in obj.items()) - if isinstance(obj, list): - return sorted(ordered(x) for x in obj) - else: - return obj + assert (len(output_desc.get('inputs')) == + len(expected_desc.get('inputs'))) + assert(len(output_desc.get('output-files')) == + len(expected_desc.get('output-files'))) + assert (output_desc.get('container-image').get('image') == + expected_desc.get('container-image').get('image')) From 52d87a9cf091668f292ad57c77fcce28ce4b33af Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Tue, 9 Apr 2019 22:33:18 -0400 Subject: [PATCH 27/28] added handling for lists with value choices, fixed logic to handle case when command line flag includes a 0 or 1 (seen in FNIRT) --- nipype/utils/nipype2boutiques.py | 38 ++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/nipype/utils/nipype2boutiques.py b/nipype/utils/nipype2boutiques.py index a9500a71a8..5f88239da1 100644 --- a/nipype/utils/nipype2boutiques.py +++ b/nipype/utils/nipype2boutiques.py @@ -268,15 +268,7 @@ def get_boutiques_input(inputs, interface, input_name, spec, verbose, elif handler_type == "Float": inp['type'] = "Number" elif handler_type == "Bool": - if (spec.argstr and len(spec.argstr.split("=")) > 1 and - (spec.argstr.split("=")[1] == '0' - or spec.argstr.split("=")[1] == '1')): - inp['type'] = "Number" - inp['integer'] = True - inp['minimum'] = 0 - inp['maximum'] = 1 - else: - inp['type'] = "Flag" + inp['type'] = "Flag" else: inp['type'] = "String" @@ -296,14 +288,24 @@ def get_boutiques_input(inputs, interface, input_name, spec, verbose, # TODO handle lists of lists (e.g. FSL ProbTrackX seed input) if handler_type == "List": inp['list'] = True - trait_type = type(trait_handler.item_trait.trait_type).__name__ - if trait_type == "Int": + item_type = trait_handler.item_trait.trait_type + item_type_name = type(item_type).__name__ + if item_type_name == "Int": inp['integer'] = True inp['type'] = "Number" - elif trait_type == "Float": + elif item_type_name == "Float": inp['type'] = "Number" - elif trait_type == "File": + elif item_type_name == "File": inp['type'] = "File" + elif item_type_name == "Enum": + value_choices = item_type.values + if value_choices is not None: + if all(isinstance(n, int) for n in value_choices): + inp['type'] = "Number" + inp['integer'] = True + elif all(isinstance(n, float) for n in value_choices): + inp['type'] = "Number" + inp['value-choices'] = value_choices else: inp['type'] = "String" if trait_handler.minlen != 0: @@ -532,7 +534,7 @@ def generate_custom_inputs(desc_inputs): for desc_input in desc_inputs: if desc_input['type'] == 'Flag': custom_input_dicts.append({desc_input['id']: True}) - elif desc_input.get('value-choices'): + elif desc_input.get('value-choices') and not desc_input.get('list'): for value in desc_input['value-choices']: custom_input_dicts.append({desc_input['id']: value}) return custom_input_dicts @@ -578,8 +580,12 @@ def get_command_line_flag(input_spec, is_flag_type=False, input_name=None): flag, flag_sep = None, None if input_spec.argstr: if "=" in input_spec.argstr: - flag = input_spec.argstr.split("=")[0].strip() - flag_sep = "=" + if (input_spec.argstr.split("=")[1] == '0' + or input_spec.argstr.split("=")[1] == '1'): + flag = input_spec.argstr + else: + flag = input_spec.argstr.split("=")[0].strip() + flag_sep = "=" elif input_spec.argstr.split("%")[0]: flag = input_spec.argstr.split("%")[0].strip() elif is_flag_type: From 5c0684d77c712ed115376b2884cdaa956b5539d1 Mon Sep 17 00:00:00 2001 From: Erin Benderoff Date: Wed, 24 Apr 2019 16:47:12 -0400 Subject: [PATCH 28/28] removed six, changed all-or-none groups to go under requires-inputs field instead --- nipype/utils/nipype2boutiques.py | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/nipype/utils/nipype2boutiques.py b/nipype/utils/nipype2boutiques.py index 5f88239da1..4f692a267b 100644 --- a/nipype/utils/nipype2boutiques.py +++ b/nipype/utils/nipype2boutiques.py @@ -19,7 +19,6 @@ import os import sys import simplejson as json -import six from ..scripts.instance import import_module @@ -310,7 +309,7 @@ def get_boutiques_input(inputs, interface, input_name, spec, verbose, inp['type'] = "String" if trait_handler.minlen != 0: inp['min-list-entries'] = trait_handler.minlen - if trait_handler.maxlen != six.MAXSIZE: + if trait_handler.maxlen != sys.maxsize: inp['max-list-entries'] = trait_handler.maxlen if spec.sep: inp['list-separator'] = spec.sep @@ -353,6 +352,8 @@ def get_boutiques_input(inputs, interface, input_name, spec, verbose, inp['optional'] = False if spec.usedefault: inp['default-value'] = spec.default_value()[1] + if spec.requires is not None: + inp['requires-inputs'] = spec.requires try: value_choices = trait_handler.values @@ -422,7 +423,8 @@ def get_boutiques_output(outputs, name, spec, interface, tool_inputs): # Handle multi-outputs if (isinstance(output_value, list) or - type(spec.handler).__name__ == "OutputMultiObject"): + type(spec.handler).__name__ == "OutputMultiObject" or + type(spec.handler).__name__ == "List"): output['list'] = True if output_value: # Check if all extensions are the same @@ -451,32 +453,19 @@ def get_boutiques_output(outputs, name, spec, interface, tool_inputs): def get_boutiques_groups(input_traits): """ Returns a list of dictionaries containing Boutiques groups for the mutually - exclusive and all-or-none Nipype inputs. + exclusive Nipype inputs. """ desc_groups = [] - all_or_none_input_sets = [] mutex_input_sets = [] # Get all the groups for name, spec in input_traits: - if spec.requires is not None: - group_members = set([name] + list(spec.requires)) - if group_members not in all_or_none_input_sets: - all_or_none_input_sets.append(group_members) if spec.xor is not None: group_members = set([name] + list(spec.xor)) if group_members not in mutex_input_sets: mutex_input_sets.append(group_members) # Create a dictionary for each one - for i, inp_set in enumerate(all_or_none_input_sets, 1): - desc_groups.append({'id': "all_or_none_group" + - ("_" + str(i) if i != 1 else ""), - 'name': "All or none group" + - (" " + str(i) if i != 1 else ""), - 'members': list(inp_set), - 'all-or-none': True}) - for i, inp_set in enumerate(mutex_input_sets, 1): desc_groups.append({'id': "mutex_group" + ("_" + str(i) if i != 1 else ""),