diff --git a/BrainPortal/lib/boutiques_support.rb b/BrainPortal/lib/boutiques_support.rb index 9164da8b9..7eca0e015 100644 --- a/BrainPortal/lib/boutiques_support.rb +++ b/BrainPortal/lib/boutiques_support.rb @@ -24,7 +24,7 @@ # # See also the Boutiques repository: https://github.com/boutiques/boutiques # -# The modules provides one main Ruby class, BoutiquesSupport::BoutiquesDescriptor, +# The module provides one main Ruby class, BoutiquesSupport::BoutiquesDescriptor, # and several smaller data classes representing components of a descriptor. # These classes are subclasses of RestrictedHash, a type of Hash class that # only recognize a select set of keys and raise an exception when other keys @@ -58,49 +58,6 @@ # appear in RDOC-generated documentation. Among these, many are # used by the BoutiquesTask integrator. # -# # Returns the name of the tool NNNN, appropriate to use as -# # a class name as BoutiquesTask::NNNN -# desc.name_as_ruby_class -# -# # Returns all tags as a flat array -# desc.flat_tag_list -# -# # Finds a specific BoutiquesSupport:Input by ID -# desc.input_by_id(inputid) -# -# # Subset of the list of inputs with just the optional ones -# desc.optional_inputs -# -# # Subset of the list of inputs with just the mandatory ones -# desc.required_inputs -# -# # Subset of the list of inputs with just the multi-valued ones -# desc.list_inputs -# -# # Subset of the list of inputs with just the File inputs -# desc.file_inputs -# -# # List of File inputs that are optional -# desc.optional_file_inputs -# -# # List of File inputs that are mandatory -# desc.required_file_inputs -# -# # Returns the entry for a custom Boutiques integration module -# desc.custom_module_info(modulename) -# -# # Utility for building a replacement hash for the inputs based on -# # the values in invoke_structure -# desc.build_substitutions_by_tokens_hash(invoke_structure) -# -# # Utility to perform the subsitutions of tokens in a string -# desc.apply_substitutions(string, substitutions_by_tokens, to_strip=[]) -# -# # Returns a new descriptor with the attributes in a canonical beautiful order -# desc.pretty_ordered -# -# # Generates a JSON with nice spacing -# desc.super_pretty_json # module BoutiquesSupport @@ -165,6 +122,7 @@ def initialize(hash={}) self end + # Creates a new Boutiques object from string def self.new_from_string(text) json = JSON.parse(text) errors = BoutiquesSupport.validate(json) @@ -173,13 +131,14 @@ def self.new_from_string(text) cb_error "Invalid Boutiques descriptor\n" + (errors.map { |e| e[:message] }.join("\n")) end + # Creates a new Boutiques object from a documents stored in a given path def self.new_from_file(path) obj = self.new_from_string(File.read(path)) obj.from_file = path obj end - def validate + def validate #:nodoc: BoutiquesSupport.validate(self) # amazingly, the JSON validator also work with our descriptor class end @@ -226,6 +185,7 @@ def name_as_ruby_class .camelize end + # Returns all tags as a flat arra def flat_tag_list tags = self.tags return [] if ! tags @@ -235,31 +195,38 @@ def flat_tag_list end.flatten end + # Finds a specific Input by id def input_by_id(inputid) inputs.detect { |x| x.id == inputid } or cb_error "No input found with ID '#{inputid}'" end + # Lists optional inputs def optional_inputs inputs.select { |x| x.optional } end + # Lists required inputs def required_inputs inputs.select { |x| ! x.optional } end + # Lists inputs def list_inputs inputs.select { |x| x.list } end + # Lists File inputs def file_inputs inputs.select { |x| x.type == 'File' } end + # Lists optional File inputs def optional_file_inputs file_inputs.select { |x| x.optional } end + # Lists mandatory File inputs def required_file_inputs file_inputs.select { |x| ! x.optional } end @@ -338,7 +305,7 @@ def build_substitutions_by_tokens_hash(invoke_structure) end.compact.to_h end - # Replaces in +string+ all occurences of the keys in + # Replaces in +string+ all occurrences of the keys in # +substitutions_by_tokens+ by the associated values. # This is typically used to build a templated string # using the "value-key" of the inputs of the descriptor. @@ -519,6 +486,136 @@ def super_pretty_json new_json end + #------------------------------------------------------------------------- + # Methods to access and document CBRAIN specific custom properties + #------------------------------------------------------------------------- + # see public/doc/boutiques_extensions for a list of these custom properties + + # Returns a string with name(s) and emails(s) of the Boutiques descriptor authors, enlisted in + # "cbrain:author" custom property of the descriptors. Emails are optional + # and should be in angle brackets + # + # For example, given the descriptor with + # + # "custom": { "cbrain:author": "Full Name , Co-author Name " } + # + # The method returns string + # "Full Name , Co-author Name " + def custom_author + authors = self.custom['cbrain:author'] + return authors if authors is_a? String + return authors.join(", ") # if author field is arrays + end + + # Returns Boutiques CBRAIN custom property indicating + # are forking sub-task(s) allowed. To submit a subtask, a task must create a JSON file + # named ".new-task-*.json" in the root of its + # work directory. An example of property definition in a tool descriptor: + # + # "custom: { + # "cbrain:can-submit-new-tasks": true + # } + def custom_can_submit_new_tasks + return self.custom["cbrain:can-submit-new-tasks"] + end + + # Returns Boutiques CBRAIN custom property indicating + # the outputs which will not be saved. + # An example of property definition in a tool descriptor: + # + # "custom: { + # "cbrain:ignore_outputs": [output_id_1, output_id_2, output_id_3 ... ] + # } + def custom_ignore_outputs + return self.custom["cbrain:ignore_outputs"] + end + + # Returns Boutiques CBRAIN custom property indicating + # inputs which are saved back to the dataprovider + # (the original data will be mutated). + # + # An example of property definition in a tool descriptor: + # "custom: { + # "cbrain:save_back_inputs": [id_1, id_2, id_3 ...] + # } + def custom_save_back_inputs + return self.custom["cbrain:save_back_inputs"] + end + + # Returns Boutiques CBRAIN custom property indicating + # that the tool does not modify inputs. + # An example of property definition in a tool descriptor: + # + # "custom: { + # "cbrain:readonly-input-files": true + # } + def custom_readonly_input_files + return self.custom["cbrain:readonly-input-files"] + end + + # Returns Boutiques CBRAIN custom property indicating + # if this task may alter its input files. + # An example of property definition in a tool descriptor: + # + # "custom: { + # "cbrain:alters-input-files": true + # } + def custom_alters_input_files + return self.custom["cbrain:alters-input-files"] + end + + # Returns Boutiques CBRAIN custom property indicating for which outputs + # the usual practice of adding a run id to output file names is cancelled, + # list of output IDs where no run id inserted. Only allowed for MultiLevel + # data-providers with "browse path" capability. + # For listed outputs ids new results overwrite old files. + # An example of property definition in a tool descriptor: + # + # "custom: { + # "cbrain:no-run-id-for-outputs": "id_1, id_2, id_3 .." + # } + def custom_no_run_id_for_outputs + return self.custom["cbrain:no-run-id-for-outputs"] + end + + # Returns Boutiques CBRAIN custom property indicating + # for which inputs an empty string is a valid input. + # An example of property definition in a tool descriptor: + # + # "custom: { + # "cbrain:allow_empty_strings": [input_id] + # } + def custom_allow_empty_strings + return self.custom["cbrain:allow_empty_strings"] + end + + # Experimental feature that affects the way tasks are executed. + # The default implied value is 'simulate' + # In the mode 'simulate', at the moment of creating + # the tool's script in cluster_commands(), the + # output of 'bosh exec simulate' will be substituted in + # the script to generate the tool's command. + # In the mode 'launch', an actual 'bosh exec launch' command + # will be put in the script instead. + # An example of property definition in a tool descriptor: + # + # "custom: { + # "cbrain:boutiques_bosh_exec_mode": "launch" + # } + def custom_boutiques_bosh_exec_mode + return self.custom["cbrain:boutiques_bosh_exec_mode"] + end + + # An advanced feature for seasoned CBRAIN experts only. That allows + # overwrite the standard task behavior with custom class. + # An example of property definition in a tool descriptor: + # "custom: { + # "cbrain:inherits-from-class": "MyClassName" + # } + def custom_inherits_from_class + return self.custom["cbrain:inherits-from-class"] + end + end # class BoutiquesSupport::BoutiquesDescriptor #------------------------------------------------------ @@ -543,23 +640,23 @@ def dup #:nodoc: copy end - # This method return the parameter name for an input identified + # This method returns the parameter name for an input identified # by input_id. # We put all input Boutiques parameters under a 'invoke' substructure. # E.g. for a input with ID 'abcd' in a task, we'll find the value # in task.params['invoke']['abcd'] and the parameter name is thus # "invoke[abcd]". The as_list option appends "[]" to the name # to make it an array parameter. - def self.cb_invoke_name(input_id, as_list = nil) + def self.cb_invoke_name(input_id, as_list = nil) #:nodoc: return "invoke[#{input_id}][]" if as_list return "invoke[#{input_id}]" end - def self.cb_invoke_html_name(input_id, force_list = nil) + def self.cb_invoke_html_name(input_id, force_list = nil) #:nodoc: self.cb_invoke_name(input_id, force_list).to_la end - def self.cb_invoke_html_id(input_id, force_list = nil) + def self.cb_invoke_html_id(input_id, force_list = nil) #:nodoc: self.cb_invoke_name(input_id, force_list).to_la_id end @@ -576,12 +673,12 @@ def cb_invoke_name(force_list = nil) self.class.cb_invoke_name(self.id, as_list) end - def cb_invoke_html_name(force_list = nil) + def cb_invoke_html_name(force_list = nil) #:nodoc: as_list = (self.list && force_list.nil?) || force_list == true self.class.cb_invoke_html_name(self.id, as_list) end - def cb_invoke_html_id(force_list = nil) + def cb_invoke_html_id(force_list = nil) #:nodoc: as_list = (self.list && force_list.nil?) || force_list == true self.class.cb_invoke_html_id(self.id, as_list) end diff --git a/BrainPortal/public/doc/boutiques_custom_properties/cbrain_properties.html b/BrainPortal/public/doc/boutiques_custom_properties/cbrain_properties.html new file mode 100644 index 000000000..51e981f17 --- /dev/null +++ b/BrainPortal/public/doc/boutiques_custom_properties/cbrain_properties.html @@ -0,0 +1,250 @@ + + + + TABLE OF CBRAIN SPECIFIC BOUTIQUES CUSTOM PROPERTIES + + + + + + +The standard Boutiques properties set does not cover all the need for CBRAIN +thus we introduce a number of custom properties, which can be added to the +"custom" section of a Boutiques descriptor, to fine-tune the way CBRAIN +interpret the descriptor. For example, + +
+    "custom":{
+        "cbrain:readonly-input-files":true,
+        "cbrain:author":"Erik Lee ",
+        "cbrain:allow_empty_strings":[
+            "derivatives_prefix"
+        ],
+        "cbrain:no-run-id-for-outputs":[
+            "OutputDirectory"
+        ],
+        "cbrain:integrator_modules":{
+            "BoutiquesFileTypeVerifier":{
+                "SubjectDirectory":[
+                    "BidsSubject"
+                ]
+            },
+            "BoutiquesFileNameMatcher":{
+                "SubjectDirectory":"^sub-[a-zA-Z0-9_]+$"
+            },
+            "BoutiquesOutputFileTypeSetter":{
+                "OutputDirectory":"MADEOutput"
+            },
+            "BoutiquesForcedOutputBrowsePath":{
+                "OutputDirectory":"[DERIVATIVES_PREFIX]made"
+            },
+            "BoutiquesBidsSingleSubjectMaker":"SubjectDirectory",
+            "BoutiquesBidsSubjectFileSelector":{
+                "SubjectDirectory":"all_to_keep"
+            }
+        }
+    }
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
JSON FragmentDescription
+ 
+"cbrain:author":
+  "Full Name <email@address>"
+
+ 		+ Author(s) of boutiques descriptor +
+ 
+"cbrain:can-submit-new-tasks":
+  true
+
+ 		+ Allows forking sub-task(s). To submit a subtask, a task must create a ".new-task-*.json" JSON file at the root of its work directory +
+ 
+"cbrain:ignore_outputs":
+  [
+    output_id_1,
+    output_id_2,
+    output_id_3,
+    ...
+  ]
+
+ 		+ The listed outputs will not be saved. +
+ 
+"cbrain:save_back_inputs":
+  [
+    id_1,
+    id_2,
+    id_3,
+    ...
+  ]
+
+ 		+ Saves back listed inputs to the dataprovider (mutates the original inputs) +
+ 
+"cbrain:readonly-input-files":
+  true
+
+ 		+ Indicates that the tool cannot modify inputs +
+ 
+"cbrain:alters-input-files":
+  true
+
+ 		+ Indicates that this task may alter its input files +
+ 
+"cbrain:no-run-id-for-outputs":
+  [
+    id_1,
+    id_2,
+    id_3,
+    ...
+  ]
+
+ 		+ Lists output IDs where no run id is inserted. Prevents the usual practice of adding a run id to output file names. Only allowed for MultiLevel data-providers with "browse path" capability. With this option, new results can overwrite old files. +
+ 
+"cbrain:allow_empty_strings":
+  [input_id]
+
+ 		+ Allow an empty string as a valid input +
+ 
+"cbrain:boutiques_bosh_exec_mode":
+  "launch"
+
+ 		+ Experimental. The default implied value is 'simulate'. In the mode 'simulate', at the moment of creating the tool's script in `cluster_commands()`, the output of 'bosh exec simulate' will be substituted in the script to generate the tool's command. In the mode 'launch', an actual 'bosh exec launch' command will be put in the script instead. +
+ 
+"cbrain:inherits-from-class":
+  "MyClassName"
+
+ 		+ An advanced feature for seasoned CBRAIN experts only. That allows overwriting the standard task behavior with a custom class. +
+ 
+"cbrain:integrator_modules":
+  {
+    "BoutiquesModuleOne":
+       { module-specific-parameters },
+    "BoutiquesModuleTwo":
+       { module-specific-parameters }
+  }
+
+ 		+ Loads a module to modify the CBRAIN behavior. Does not work with legacy integration method. All the modules are defined in the lib folder of CBRAIN codebase or a plugin. Thus documentation for all the module-based sub-properties can be auto-generated with the `rdoc` utility. +
+ 
+"cbrain:ignore-exit-status":
+  true
+
+ 		+ Deprecated. Considers the task successful even if the wrong exit status is present. For the case where the tool has wrong exit codes (deprecated, being superseded by a module-based property). +
+ 
+"cbrain:walltime-estimate":
+  value_in_seconds
+
+ 		+ Deprecated since walltime was added to Boutiques resources sections. +
+ + + diff --git a/BrainPortal/public/doc/boutiques_custom_properties/cbrain_properties.txt b/BrainPortal/public/doc/boutiques_custom_properties/cbrain_properties.txt new file mode 100644 index 000000000..466367554 --- /dev/null +++ b/BrainPortal/public/doc/boutiques_custom_properties/cbrain_properties.txt @@ -0,0 +1,116 @@ +TABLE OF CBRAIN SPECIFIC BOUTIQUES CUSTOM PROPERTIES + +The standard Boutiques properties set does not cover all the need for CBRAIN +thus we introduce a number of custom properties, which can be added to the +"custom" section of a Boutiques descriptor, to fine-tune the way CBRAIN +interpret the descriptor. For example, + + + "custom":{ + "cbrain:readonly-input-files":true, + "cbrain:author":"Erik Lee ", + "cbrain:allow_empty_strings":[ + "derivatives_prefix" + ], + "cbrain:no-run-id-for-outputs":[ + "OutputDirectory" + ], + "cbrain:integrator_modules":{ + "BoutiquesFileTypeVerifier":{ + "SubjectDirectory":[ + "BidsSubject" + ] + }, + "BoutiquesFileNameMatcher":{ + "SubjectDirectory":"^sub-[a-zA-Z0-9_]+$" + }, + "BoutiquesOutputFileTypeSetter":{ + "OutputDirectory":"MADEOutput" + }, + "BoutiquesForcedOutputBrowsePath":{ + "OutputDirectory":"[DERIVATIVES_PREFIX]made" + }, + "BoutiquesBidsSingleSubjectMaker":"SubjectDirectory", + "BoutiquesBidsSubjectFileSelector":{ + "SubjectDirectory":"all_to_keep" + } + } + } + + ++-----------------------------------+----------------------------------------------------------------------------------+ +| PROPERTY | DESCRIPTION | ++-----------------------------------+----------------------------------------------------------------------------------+ +| "cbrain:author": | | +| "Full Name " | Author(s) of Boutiques descriptor | ++-----------------------------------+----------------------------------------------------------------------------------+ +| "cbrain:can-submit-new-tasks": | Experimental | +| true | Allows forking sub-task(s). To submit a subtask, a task must create | +| | a ".new-task-*.json" JSON file at the root of its work directory | ++-----------------------------------+----------------------------------------------------------------------------------+ +| "cbrain:ignore_outputs": [ | | +| output_id_1, | The listed outputs will not be saved. | +| output_id_2, | | +| output_id_3, | | +| ... | | +| ] | | +| | | ++-----------------------------------+----------------------------------------------------------------------------------+ +| "cbrain:save_back_inputs": [ | | +| id_1, | Saves back listed inputs to the dataprovider (mutates the original inputs) | +| id_2, | | +| id_3, | | +| ... | | +| ] | | +| | | ++-----------------------------------+----------------------------------------------------------------------------------+ +| "cbrain:readonly-input-files": | | +| true | Indicates that the tool cannot modify inputs | ++-----------------------------------+----------------------------------------------------------------------------------+ +| "cbrain:alters-input-files": | | +| true | Indicates that this task may alter its input files | ++-----------------------------------+----------------------------------------------------------------------------------+ +| "cbrain:no-run-id-for-outputs": [ | | +| id_1, | Lists output IDs where no run id is inserted. | +| id_2, | Prevents the usual practice of adding a run id to output file names. | +| id_3, | Only allowed for MultiLevel data-providers with "browse path" capability. | +| ] | With this option, new results can overwrite old files. | +| | | +| | | ++-----------------------------------+----------------------------------------------------------------------------------+ +| "cbrain:allow_empty_strings": | | +| [input_id_1, input_id_2 ...] | Allow an empty string as a valid input | +| | | ++-----------------------------------+----------------------------------------------------------------------------------+ +| "cbrain:boutiques_bosh_exec_mode":| (Experimental) | +| "launch" | The default implied value is 'simulate'. | +| | In the mode 'simulate', at the moment of creating the | +| | tool's script in cluster_commands(), the output of | +| | bosh exec simulate | +| | will be substituted in the script to | +| | generate the tool's command. In the mode 'launch', an actual | +| | bosh exec launch | +| | command will be put in the script instead. | ++-----------------------------------+----------------------------------------------------------------------------------+ +| "cbrain:inherits-from-class": | ( Restricted use ) | +| "MyClassName" | An advanced feature for seasoned CBRAIN experts only. | +| | That allows overwriting the standard task behavior with a custom class. | ++-----------------------------------+----------------------------------------------------------------------------------+ +| "cbrain:integrator_modules": { | Loads a module to modify the CBRAIN behavior. | +| | Does not work for legacy integration method | +| "BoutiquesModuleOne": | All the modules are defined in the lib folder of CBRAIN | +| {module-specific-parameters}, | codebase or a plugin. Thus documentation for all the | +| "BoutiquesModuleTwo": | module-based sub-properties can be auto-generated | +| {module-specific-parameters} | with the rdoc utility. | +| } | | +| | | ++-----------------------------------+----------------------------------------------------------------------------------+ +| "cbrain:ignore-exit-status": | (Deprecated) | +| true | Considers the task successful even if there is a wrong exit status. | +| | For the case the tool has wrong exit codes | +| | (deprecated and being superseded by a module-based property). | +| | | ++-----------------------------------+----------------------------------------------------------------------------------+ +| "cbrain:walltime-estimate": | (Deprecated) | +| value_in_seconds | Since walltime was added to Boutiques resources sections. | ++-----------------------------------+----------------------------------------------------------------------------------+