🐛 Fix parsing FROM: /filepath

CHANGELOG.md

@@ -32,6 +32,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Handling of `3dECM` outputs for AFNI ≥ 21.1.1.
 - Fixed a bug where sparsity thresholds were not being scaled for network centrality.
 - Fixed a bug where `calculate_motion_first` would not calculate motion at all.
+- Fixed a bug in parsing `FROM: /file/path` syntax
 
 ### Changed
 

CPAC/utils/configuration/configuration.py

@@ -21,6 +21,7 @@
 from typing import Optional
 from warnings import warn
 import pkg_resources as p
+from click import BadParameter
 import yaml
 from CPAC.utils.typing import ConfigKeyType, TUPLE
 from .diff import dct_diff
@@ -83,9 +84,22 @@ class Configuration:
     >>> c['pipeline_setup', 'pipeline_name'] = 'new_pipeline2'
     >>> c['pipeline_setup', 'pipeline_name']
     'new_pipeline2'
+
+    >>> from CPAC.utils.tests.configs import SLACK_420349
+
+    # test "FROM: /path/to/file"
+    >>> slack_420349_filepath = Configuration(
+    ...     yaml.safe_load(SLACK_420349['filepath']))
+    >>> slack_420349_filepath['pipeline_setup', 'pipeline_name']
+    'slack_420349_filepath'
+
+    # test "FROM: preconfig"
+    >>> slack_420349_preconfig = Configuration(
+    ...    yaml.safe_load(SLACK_420349['preconfig']))
+    >>> slack_420349_preconfig['pipeline_setup', 'pipeline_name']
+    'slack_420349_preconfig'
     """
     def __init__(self, config_map=None):
-        from click import BadParameter
         from CPAC.pipeline.schema import schema
         from CPAC.utils.utils import lookup_nested_value, update_nested_dict
 
@@ -104,8 +118,8 @@ def __init__(self, config_map=None):
             config_map = update_nested_dict(base_config.dict(), config_map)
         else:
             # base everything on blank pipeline for unspecified keys
-            with open(preconfig_yaml('blank'), 'r', encoding='utf-8') as _f:
-                config_map = update_nested_dict(yaml.safe_load(_f), config_map)
+            config_map = update_nested_dict(
+                preconfig_yaml('blank', load=True), config_map)
 
         config_map = self._nonestr_to_None(config_map)
 
@@ -604,7 +618,8 @@ def configuration_from_file(config_file):
 
 
 def preconfig_yaml(preconfig_name='default', load=False):
-    """Get the path to a preconfigured pipeline's YAML file
+    """Get the path to a preconfigured pipeline's YAML file.
+    Raises BadParameter if an invalid preconfig name is given.
 
     Parameters
     ----------
@@ -618,6 +633,13 @@ def preconfig_yaml(preconfig_name='default', load=False):
     str or dict
         path to YAML file or dict loaded from YAML
     """
+    from CPAC.pipeline import ALL_PIPELINE_CONFIGS, AVAILABLE_PIPELINE_CONFIGS
+    if preconfig_name not in ALL_PIPELINE_CONFIGS:
+        raise BadParameter(
+            f"The pre-configured pipeline name '{preconfig_name}' you "
+            "provided is not one of the available pipelines.\n\nAvailable "
+            f"pipelines:\n{str(AVAILABLE_PIPELINE_CONFIGS)}\n",
+            param='preconfig')
     if load:
         with open(preconfig_yaml(preconfig_name), 'r', encoding='utf-8') as _f:
             return yaml.safe_load(_f)

CPAC/utils/configuration/yaml_template.py

@@ -20,6 +20,7 @@
 import re
 from datetime import datetime
 from hashlib import sha1
+from click import BadParameter
 import yaml
 
 from CPAC.utils.configuration import Configuration, Preconfiguration, \
@@ -58,16 +59,20 @@ def __init__(self, original_yaml, base_config=None):
 
         base_config : Configuration, optional
         """
-        preconfig_path = preconfig_yaml(original_yaml)
-        if os.path.exists(preconfig_path):
-            original_yaml = preconfig_path
+        try:
+            original_yaml = preconfig_yaml(original_yaml)
+        except BadParameter:
+            pass
         if os.path.exists(original_yaml):
             with open(original_yaml, 'r', encoding='utf-8') as _f:
                 original_yaml = _f.read()
         self.comments = {}
         self.template = original_yaml
         if base_config is None:
-            self._dict = yaml.safe_load(self.template)
+            if isinstance(self.template, dict):
+                self._dict = self.template
+            if isinstance(self.template, str):
+                self._dict = yaml.safe_load(self.template)
         else:
             self._dict = base_config.dict()
         self._parse_comments()

CPAC/utils/tests/configs/__init__.py

@@ -1,14 +1,22 @@
 """Configs for testing"""
-import os
+from pathlib import Path
 from pkg_resources import resource_filename
-with open(resource_filename("CPAC",
-                            os.path.join('utils', 'tests', 'configs',
-                                         'neurostars_23786.yml')),
-          'r', encoding='utf-8') as _f:
+import yaml
+
+_TEST_CONFIGS_PATH = Path(resource_filename("CPAC", "utils/tests/configs"))
+with open(_TEST_CONFIGS_PATH / "neurostars_23786.yml", "r", encoding="utf-8"
+          ) as _f:
+    # A loaded YAML file to test https://tinyurl.com/neurostars23786
     NEUROSTARS_23786 = _f.read()
-with open(resource_filename("CPAC",
-                            os.path.join('utils', 'tests', 'configs',
-                                         'neurostars_24035.yml')),
-          'r', encoding='utf-8') as _f:
+with open(_TEST_CONFIGS_PATH / "neurostars_24035.yml", "r", encoding="utf-8"
+          ) as _f:
+    # A loaded YAML file to test https://tinyurl.com/neurostars24035
     NEUROSTARS_24035 = _f.read()
-__all__ = ['NEUROSTARS_23786', 'NEUROSTARS_24035']
+# A loaded YAML file to test https://tinyurl.com/cmicnlslack420349
+SLACK_420349 = {
+    "filepath": yaml.dump({
+        "FROM": str(_TEST_CONFIGS_PATH / "neurostars_23786.yml"),
+        "pipeline_setup": {"pipeline_name": "slack_420349_filepath"}}),
+    "preconfig": yaml.dump({"FROM": "preproc",
+        "pipeline_setup": {"pipeline_name": "slack_420349_preconfig"}})}
+__all__ = ["NEUROSTARS_23786", "NEUROSTARS_24035", "SLACK_420349"]

CPAC/utils/tests/test_yaml.py

@@ -50,8 +50,7 @@ def test_yaml_template():
     # Create a new YAML configuration file based on the default pipeline
     # YAML file.
     pipeline_file = preconfig_yaml('default')
-    with open(pipeline_file, 'r', encoding='utf-8') as f:
-        config = yaml.safe_load(f)
+    config = preconfig_yaml('default', load=True)
     new_config = create_yaml_from_template(config, pipeline_file)
     with open(config_file, 'w', encoding='utf-8') as f:
         f.write(new_config)

CPAC/utils/utils.py

@@ -25,11 +25,9 @@
 import numpy as np
 import yaml
 
-from click import BadParameter
 from copy import deepcopy
 from itertools import repeat
 from voluptuous.error import Invalid
-from CPAC.pipeline import ALL_PIPELINE_CONFIGS, AVAILABLE_PIPELINE_CONFIGS
 
 CONFIGS_DIR = os.path.abspath(os.path.join(
     __file__, *repeat(os.path.pardir, 2), 'resources/configs/'))
@@ -1515,33 +1513,6 @@ def delete_nested_value(d, keys):
     return d
 
 
-def load_preconfig(pipeline_label):
-    import os
-    import pkg_resources as p
-
-    if pipeline_label not in ALL_PIPELINE_CONFIGS:
-        raise BadParameter(
-            "The pre-configured pipeline name '{0}' you provided is not one "
-            "of the available pipelines.\n\nAvailable pipelines:\n"
-            "{1}\n".format(pipeline_label, str(AVAILABLE_PIPELINE_CONFIGS)),
-            param='preconfig')
-
-    pipeline_file = \
-        p.resource_filename(
-            "CPAC",
-            os.path.join(
-                "resources",
-                "configs",
-                "pipeline_config_{0}.yml".format(pipeline_label)
-            )
-        )
-
-    print(f"Loading the '{pipeline_label}' pre-configured pipeline.",
-          file=sys.stderr)
-
-    return pipeline_file
-
-
 def ordereddict_to_dict(value):
     '''
     this function convert ordereddict into regular dict

dev/docker_data/run.py

@@ -40,7 +40,7 @@
 from CPAC.utils.configuration.yaml_template import create_yaml_from_template, \
                                                    hash_data_config, \
                                                    upgrade_pipeline_to_1_8
-from CPAC.utils.utils import load_preconfig, update_nested_dict
+from CPAC.utils.utils import update_nested_dict
 simplefilter(action='ignore', category=FutureWarning)
 logger = logging.getLogger('nipype.workflow')
 DEFAULT_TMP_DIR = "/tmp"
@@ -460,11 +460,14 @@ def run_main():
                 run(f"bids-validator {bids_dir}")
 
         if args.preconfig:
-            args.pipeline_file = load_preconfig(args.preconfig)
+            args.pipeline_file = preconfig_yaml(args.preconfig)
 
         # otherwise, if we are running group, participant, or dry run we
         # begin by conforming the configuration
-        c = load_yaml_config(args.pipeline_file, args.aws_input_creds)
+        if isinstance(args.pipeline_file, dict):
+            c = args.pipeline_file
+        else:
+            c = load_yaml_config(args.pipeline_file, args.aws_input_creds)
 
         if 'pipeline_setup' not in c:
             _url = (f'{DOCS_URL_PREFIX}/user/pipelines/'