Merge pull request #481 from nipreps/enh/multiverse-mode

enh: add multiverse option to output layout

Author: mgxd

nibabies/cli/parser.py

@@ -619,11 +619,12 @@ def _str_none(val):
         '--output-layout',
         action='store',
         default='bids',
-        choices=('bids', 'legacy'),
+        choices=('bids', 'legacy', 'multiverse'),
         help='Organization of outputs. bids (default) places NiBabies derivatives '
         'directly in the output directory, and defaults to placing FreeSurfer '
         'derivatives in <output-dir>/sourcedata/freesurfer. legacy creates derivative '
-        'datasets as subdirectories of outputs.',
+        'datasets as subdirectories of outputs. multiverse appends the version and a hash '
+        'of parameters used to the output folder - the hash is also applied to the output files.',
     )
     g_other.add_argument(
         '-w',
@@ -834,27 +835,62 @@ def parse_args(args=None, namespace=None):
 applied."""
         )
 
+    config.workflow.skull_strip_template = config.workflow.skull_strip_template[0]
+
     bids_dir = config.execution.bids_dir
     output_dir = config.execution.output_dir
     work_dir = config.execution.work_dir
     version = config.environment.version
     output_layout = config.execution.output_layout
+    config.execution.parameters_hash = config.hash_config(config.get())
 
-    if config.execution.fs_subjects_dir is None:
-        if output_layout == 'bids':
-            config.execution.fs_subjects_dir = output_dir / 'sourcedata' / 'freesurfer'
-        elif output_layout == 'legacy':
-            config.execution.fs_subjects_dir = output_dir / 'freesurfer'
+    # Multiverse behaves as a cross between bids and legacy
     if config.execution.nibabies_dir is None:
-        if output_layout == 'bids':
-            config.execution.nibabies_dir = output_dir
-        elif output_layout == 'legacy':
-            config.execution.nibabies_dir = output_dir / 'nibabies'
+        match output_layout:
+            case 'bids':
+                config.execution.nibabies_dir = output_dir
+            case 'legacy':
+                config.execution.nibabies_dir = output_dir / 'nibabies'
+            case 'multiverse':
+                config.loggers.cli.warning(
+                    'Multiverse output selected - assigning output directory based on version'
+                    ' and configuration hash.'
+                )
+                config.execution.nibabies_dir = (
+                    output_dir
+                    / f'nibabies-{version.split("+", 1)[0]}-{config.execution.parameters_hash}'
+                )
+            case _:
+                config.loggers.cli.warning('Unknown output layout %s', output_layout)
+                pass
+
+    nibabies_dir = config.execution.nibabies_dir
+
+    if config.execution.fs_subjects_dir is None:
+        match output_layout:
+            case 'bids':
+                config.execution.fs_subjects_dir = output_dir / 'sourcedata' / 'freesurfer'
+            case 'legacy':
+                config.execution.fs_subjects_dir = output_dir / 'freesurfer'
+            case 'multiverse':
+                config.execution.fs_subjects_dir = (
+                    nibabies_dir / 'sourcedata' / f'freesurfer-{config.execution.parameters_hash}'
+                )
+            case _:
+                pass
+
     if config.workflow.surface_recon_method == 'mcribs':
-        if output_layout == 'bids':
-            config.execution.mcribs_dir = output_dir / 'sourcedata' / 'mcribs'
-        elif output_layout == 'legacy':
-            config.execution.mcribs_dir = output_dir / 'mcribs'
+        match output_layout:
+            case 'bids':
+                config.execution.mcribs_dir = output_dir / 'sourcedata' / 'mcribs'
+            case 'legacy':
+                config.execution.mcribs_dir = output_dir / 'mcribs'
+            case 'multiverse':
+                config.execution.mcribs_dir = (
+                    nibabies_dir / 'sourcedata' / f'mcribs-{config.execution.parameters_hash}'
+                )
+            case _:
+                pass
         # Ensure the directory is created
         config.execution.mcribs_dir.mkdir(exist_ok=True, parents=True)
 
@@ -909,7 +945,6 @@ def parse_args(args=None, namespace=None):
         participant_ids=config.execution.participant_label,
         session_ids=config.execution.session_id,
     )
-    config.workflow.skull_strip_template = config.workflow.skull_strip_template[0]
 
     # finally, write config to file
     config_file = config.execution.work_dir / config.execution.run_uuid / 'config.toml'

nibabies/cli/run.py

@@ -144,16 +144,20 @@ def main():
         finally:
             from ..reports.core import generate_reports
 
+            add_hash = config.execution.output_layout == 'multiverse'
+
             # Generate reports phase
             generate_reports(
                 config.execution.unique_labels,
                 config.execution.nibabies_dir,
                 config.execution.run_uuid,
+                config_hash=config.execution.parameters_hash if add_hash else None,
             )
             write_derivative_description(
                 config.execution.bids_dir,
                 config.execution.nibabies_dir,
                 config.execution.dataset_links,
+                config.execution.parameters_hash,
             )
             write_bidsignore(config.execution.nibabies_dir)
 

nibabies/config.py

@@ -89,6 +89,7 @@
 """
 
 import os
+import typing as ty
 from multiprocessing import set_start_method
 
 from templateflow.conf import TF_LAYOUT
@@ -409,6 +410,8 @@ class execution(_Config):
     output_spaces = None
     """List of (non)standard spaces designated (with the ``--output-spaces`` flag of
     the command line) as spatial references for outputs."""
+    parameters_hash = None
+    """Unique hash of the current configuration parameters."""
     reference_anat = None
     """Force usage of this anatomical scan as the structural reference."""
     reports_only = False
@@ -792,15 +795,79 @@ def _process_initializer(cwd, omp_nthreads):
     os.environ['OMP_NUM_THREADS'] = f'{omp_nthreads}'
 
 
-def dismiss_echo(entities: list | None = None):
+def dismiss_entities(entities: list | None = None) -> list:
     """Set entities to dismiss in a DerivativesDataSink."""
     from niworkflows.utils.connections import listify
 
-    entities = entities or []
+    entities = set(entities or [])
     echo_idx = execution.echo_idx
     if echo_idx is None or len(listify(echo_idx)) > 2:
-        entities.append('echo')
-    return entities
+        entities.add('echo')
+    output_layout = execution.output_layout
+    if output_layout != 'multiverse':
+        entities.add('hash')
+    return list(entities)
+
+
+DEFAULT_DISMISS_ENTITIES = dismiss_entities()
+
+DEFAULT_CONFIG_HASH_FIELDS = {
+    'execution': [
+        'sloppy',
+        'echo_idx',
+        'reference_anat',
+    ],
+    'workflow': [
+        'surface_recon_method',
+        'bold2anat_dof',
+        'bold2anat_init',
+        'dummy_scans',
+        'fd_radius',
+        'fmap_bspline',
+        'fmap_demean',
+        'force_syn',
+        'hmc_bold_frame',
+        'longitudinal',
+        'medial_surface_nan',
+        'multi_step_reg',
+        'norm_csf',
+        'project_goodvoxels',
+        'regressors_dvars_th',
+        'regressors_fd_th',
+        'skull_strip_fixed_seed',
+        'skull_strip_template',
+        'skull_strip_anat',
+        'slice_time_ref',
+        'surface_recon_method',
+        'use_bbr',
+        'use_syn_sdc',
+        'me_t2s_fit_method',
+    ],
+}
+
+
+def hash_config(
+    conf: dict[str, ty.Any],
+    *,
+    fields_required: dict[str, list[str]] = DEFAULT_CONFIG_HASH_FIELDS,
+    version: str = None,
+    digest_size: int = 4,
+) -> str:
+    """
+    Generate a unique BLAKE2b hash of configuration attributes.
+
+    By default, uses a preselected list of workflow-altering parameters.
+    """
+    import json
+    from hashlib import blake2b
+
+    if version is None:
+        from nibabies import __version__ as version
 
+    data = {}
+    for level, fields in fields_required.items():
+        for f in fields:
+            data[f] = conf[level].get(f, None)
 
-DEFAULT_DISMISS_ENTITIES = dismiss_echo()
+    datab = json.dumps(data, sort_keys=True).encode()
+    return blake2b(datab, digest_size=digest_size).hexdigest()

nibabies/reports/core.py

@@ -10,9 +10,9 @@ def run_reports(
     subject,
     run_uuid,
     session=None,
-    out_filename=None,
+    bootstrap_file=None,
+    out_filename='report.html',
     reportlets_dir=None,
-    packagename=None,
 ):
     """
     Run the reports.
@@ -22,17 +22,20 @@ def run_reports(
         run_uuid,
         subject=subject,
         session=session,
-        bootstrap_file=load_data.readable('reports-spec.yml'),
+        bootstrap_file=load_data('reports-spec.yml'),
         reportlets_dir=reportlets_dir,
+        out_filename=out_filename,
     ).generate_report()
 
 
 def generate_reports(
     sub_ses_list,
     output_dir,
     run_uuid,
+    *,
     work_dir=None,
-    packagename=None,
+    bootstrap_file=None,
+    config_hash=None,
 ):
     """Execute run_reports on a list of subjects."""
     reportlets_dir = None
@@ -41,14 +44,23 @@ def generate_reports(
 
     report_errors = []
     for subject, session in sub_ses_list:
+        # Determine the output filename
+        html_report = f'sub-{subject}'
+        if session is not None:
+            html_report += f'_ses-{session}'
+        if config_hash is not None:
+            html_report += f'_{config_hash}'
+        html_report += '.html'
+
         report_errors.append(
             run_reports(
                 output_dir,
                 subject,
                 run_uuid,
-                session=session,
-                packagename=packagename,
+                bootstrap_file=bootstrap_file,
                 reportlets_dir=reportlets_dir,
+                out_filename=html_report,
+                session=session,
             )
         )
 

nibabies/tests/test_config.py

@@ -146,3 +146,19 @@ def _load_spaces(age):
     # Conditional based on workflow necessities
     spaces = init_workflow_spaces(init_execution_spaces(), age)
     return spaces
+
+
+def test_hash_config():
+    # This may change with changes to config defaults / new attributes!
+    expected = 'cfee5aaf'
+    assert config.hash_config(config.get()) == expected
+    _reset_config()
+
+    config.execution.log_level = 5  # non-vital attributes do not matter
+    assert config.hash_config(config.get()) == expected
+    _reset_config()
+
+    # but altering a vital attribute will create a new hash
+    config.workflow.surface_recon_method = 'mcribs'
+    assert config.hash_config(config.get()) != expected
+    _reset_config()

nibabies/utils/bids.py

@@ -40,7 +40,7 @@ def write_bidsignore(deriv_dir):
     ignore_file.write_text('\n'.join(bids_ignore) + '\n')
 
 
-def write_derivative_description(bids_dir, deriv_dir, dataset_links=None):
+def write_derivative_description(bids_dir, deriv_dir, dataset_links=None, config_hash=None):
     from nibabies import __version__
 
     DOWNLOAD_URL = f'https://github.com/nipreps/nibabies/archive/{__version__}.tar.gz'
@@ -56,6 +56,7 @@ def write_derivative_description(bids_dir, deriv_dir, dataset_links=None):
                 'Name': 'NiBabies',
                 'Version': __version__,
                 'CodeURL': DOWNLOAD_URL,
+                'ConfigurationHash': config_hash,
             }
         ],
         'HowToAcknowledge': 'TODO',

nibabies/utils/derivatives.py

@@ -136,6 +136,7 @@ def copy_derivatives(
     modality: str,
     subject_id: str,
     session_id: str | None = None,
+    config_hash: str | None = None,
 ) -> None:
     """
     Creates a copy of any found derivatives into output directory.
@@ -154,8 +155,19 @@ def copy_derivatives(
         if not isinstance(deriv, str):
             continue
         deriv = Path(deriv)
-
-        shutil.copy2(deriv, outpath / deriv.name)
-        json = deriv.parent / (deriv.name.split('.')[0] + '.json')
+        outname = deriv.name
+
+        if config_hash:
+            ents = outname.split('_')
+            if any(ent.startswith('hash-') for ent in ents):
+                # Avoid adding another hash
+                pass
+            else:
+                idx = 2 if ents[1].startswith('ses-') else 1
+                ents.insert(idx, f'hash-{config_hash}')
+                outname = '_'.join(ents)
+
+        shutil.copy2(deriv, outpath / outname)
+        json = deriv.parent / (outname.split('.')[0] + '.json')
         if json.exists():
             shutil.copy2(json, outpath / json.name)

nibabies/workflows/base.py

@@ -330,6 +330,9 @@ def init_single_subject_wf(
                 modality='anat',
                 subject_id=f'sub-{subject_id}',
                 session_id=f'ses-{session_id}' if session_id else None,
+                config_hash=config.execution.parameters_hash
+                if config.execution.output_layout == 'multiverse'
+                else None,
             )
 
     # Determine some session level options here, as we should have
@@ -368,7 +371,6 @@ def init_single_subject_wf(
         )
 
     anat = reference_anat.lower()  # To be used for workflow connections
-
     LOGGER.info(
         'Collected the following data for %s:\nRaw:\n%s\n\nDerivatives:\n\n%s\n',
         f'sub-{subject_id}' if not session_id else f'sub-{subject_id}_ses-{session_id}',
@@ -739,6 +741,9 @@ def init_single_subject_wf(
                     modality='func',
                     subject_id=f'sub-{subject_id}',
                     session_id=f'ses-{session_id}' if session_id else None,
+                    config_hash=config.execution.parameters_hash
+                    if config.execution.output_layout == 'multiverse'
+                    else None,
                 )
 
         bold_wf = init_bold_wf(
@@ -837,6 +842,10 @@ def clean_datasinks(workflow: pe.Workflow) -> pe.Workflow:
     for node in workflow.list_node_names():
         if node.split('.')[-1].startswith('ds_'):
             workflow.get_node(node).interface.out_path_base = ''
+            workflow.get_node(node).interface.inputs.base_directory = config.execution.nibabies_dir
+
+            if config.execution.output_layout == 'multiverse':
+                workflow.get_node(node).interface.inputs.hash = config.execution.parameters_hash
     return workflow