From e36da511e77ab80976fb0e9fa3bb4ba82426528d Mon Sep 17 00:00:00 2001 From: mathiasg Date: Fri, 15 Aug 2025 15:53:48 -0400 Subject: [PATCH 01/14] enh: add config hashing --- nibabies/config.py | 61 +++++++++++++++++++++++++++++++++++ nibabies/tests/test_config.py | 16 +++++++++ 2 files changed, 77 insertions(+) diff --git a/nibabies/config.py b/nibabies/config.py index cab424cd..594a1f44 100644 --- a/nibabies/config.py +++ b/nibabies/config.py @@ -89,6 +89,7 @@ """ import os +import typing as ty from multiprocessing import set_start_method from templateflow.conf import TF_LAYOUT @@ -804,3 +805,63 @@ def dismiss_echo(entities: list | None = None): DEFAULT_DISMISS_ENTITIES = dismiss_echo() + +DEFAULT_CONFIG_HASH_FIELDS = { + 'execution': [ + 'sloppy', + 'echo_idx', + 'reference_anat', + ], + 'workflow': [ + 'surface_recon_method', + 'bold2anat_dof', + 'bold2anat_init', + 'dummy_scans', + 'fd_radius', + 'fmap_bspline', + 'fmap_demean', + 'force_syn', + 'hmc_bold_frame', + 'longitudinal', + 'medial_surface_nanmulti_step_reg', + 'norm_csf', + 'project_goodvoxels', + 'regressors_dvars_th', + 'regressors_fd_th', + 'skull_strip_fixed_seed', + 'skull_strip_template', + 'skull_strip_anat', + 'slice_time_ref', + 'surface_recon_method', + 'use_bbr', + 'use_syn_sdc', + 'me_t2s_fit_method', + ], +} + + +def hash_config( + conf: dict[str, ty.Any], + *, + fields_required: dict[str, list[str]] = DEFAULT_CONFIG_HASH_FIELDS, + version: str = None, + digest_size: int = 4, +) -> str: + """ + Generate a unique BLAKE2b hash of configuration attributes. + + By default, uses a preselected list of workflow-altering parameters. + """ + import json + from hashlib import blake2b + + if version is None: + from nibabies import __version__ as version + + data = {} + for level, fields in fields_required.items(): + for f in fields: + data[f] = conf[level].get(f, None) + + datab = json.dumps(data, sort_keys=True).encode() + return blake2b(datab, digest_size=digest_size).hexdigest() diff --git a/nibabies/tests/test_config.py b/nibabies/tests/test_config.py index f585e912..a48c883c 100644 --- a/nibabies/tests/test_config.py +++ b/nibabies/tests/test_config.py @@ -146,3 +146,19 @@ def _load_spaces(age): # Conditional based on workflow necessities spaces = init_workflow_spaces(init_execution_spaces(), age) return spaces + + +def test_hash_config(): + # This may change with changes to config defaults / new attributes! + expected = '1fd5c50e' + assert config.hash_config(config.get()) == expected + _reset_config() + + config.execution.log_level = 5 # non-vital attributes do not matter + assert config.hash_config(config.get()) == expected + _reset_config() + + # but altering a vital attribute will create a new hash + config.workflow.surface_recon_method = 'mcribs' + assert config.hash_config(config.get()) != expected + _reset_config() From f7a0661f64c0bde663839e04b15a01b7adffb296 Mon Sep 17 00:00:00 2001 From: mathiasg Date: Tue, 19 Aug 2025 12:53:31 -0400 Subject: [PATCH 02/14] fix: separate config values --- nibabies/config.py | 3 ++- nibabies/tests/test_config.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/nibabies/config.py b/nibabies/config.py index 594a1f44..71656e4c 100644 --- a/nibabies/config.py +++ b/nibabies/config.py @@ -823,7 +823,8 @@ def dismiss_echo(entities: list | None = None): 'force_syn', 'hmc_bold_frame', 'longitudinal', - 'medial_surface_nanmulti_step_reg', + 'medial_surface_nan', + 'multi_step_reg', 'norm_csf', 'project_goodvoxels', 'regressors_dvars_th', diff --git a/nibabies/tests/test_config.py b/nibabies/tests/test_config.py index a48c883c..1f4da13e 100644 --- a/nibabies/tests/test_config.py +++ b/nibabies/tests/test_config.py @@ -150,7 +150,7 @@ def _load_spaces(age): def test_hash_config(): # This may change with changes to config defaults / new attributes! - expected = '1fd5c50e' + expected = 'cfee5aaf' assert config.hash_config(config.get()) == expected _reset_config() From afae429470281f18d7dd4c56faa1fbf42cd1f913 Mon Sep 17 00:00:00 2001 From: mathiasg Date: Fri, 22 Aug 2025 11:51:07 -0400 Subject: [PATCH 03/14] rf: dismiss hash if not running multiverse --- nibabies/config.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/nibabies/config.py b/nibabies/config.py index 71656e4c..23806f43 100644 --- a/nibabies/config.py +++ b/nibabies/config.py @@ -793,18 +793,21 @@ def _process_initializer(cwd, omp_nthreads): os.environ['OMP_NUM_THREADS'] = f'{omp_nthreads}' -def dismiss_echo(entities: list | None = None): +def dismiss_entities(entities: list | None = None) -> list: """Set entities to dismiss in a DerivativesDataSink.""" from niworkflows.utils.connections import listify - entities = entities or [] + entities = set(entities or []) echo_idx = execution.echo_idx if echo_idx is None or len(listify(echo_idx)) > 2: - entities.append('echo') - return entities + entities.add('echo') + output_layout = execution.output_layout + if output_layout != 'multiverse': + entities.add('hash') + return list(entities) -DEFAULT_DISMISS_ENTITIES = dismiss_echo() +DEFAULT_DISMISS_ENTITIES = dismiss_entities() DEFAULT_CONFIG_HASH_FIELDS = { 'execution': [ From becf37ba1e17d97661b3d21617768a56307dfa0b Mon Sep 17 00:00:00 2001 From: mathiasg Date: Fri, 22 Aug 2025 16:58:16 -0400 Subject: [PATCH 04/14] enh: add multiverse output layout, add hash to dataset desc --- nibabies/cli/parser.py | 13 ++++++++++--- nibabies/cli/run.py | 1 + nibabies/config.py | 1 + nibabies/utils/bids.py | 3 ++- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/nibabies/cli/parser.py b/nibabies/cli/parser.py index 8bb1d201..1217ad3a 100644 --- a/nibabies/cli/parser.py +++ b/nibabies/cli/parser.py @@ -619,11 +619,12 @@ def _str_none(val): '--output-layout', action='store', default='bids', - choices=('bids', 'legacy'), + choices=('bids', 'legacy', 'multiverse'), help='Organization of outputs. bids (default) places NiBabies derivatives ' 'directly in the output directory, and defaults to placing FreeSurfer ' 'derivatives in /sourcedata/freesurfer. legacy creates derivative ' - 'datasets as subdirectories of outputs.', + 'datasets as subdirectories of outputs. multiverse appends the version and a hash ' + 'of parameters used to the output folder - the hash is also applied to the output files.', ) g_other.add_argument( '-w', @@ -834,22 +835,29 @@ def parse_args(args=None, namespace=None): applied.""" ) + config.workflow.skull_strip_template = config.workflow.skull_strip_template[0] + bids_dir = config.execution.bids_dir output_dir = config.execution.output_dir work_dir = config.execution.work_dir version = config.environment.version output_layout = config.execution.output_layout + config.execution._config_hash = config.hash_config(config.get()) + if output_layout == 'multiverse': + output_dir += f'-{version.split("+", 1)[0]}-{config.execution.config_hash}' if config.execution.fs_subjects_dir is None: if output_layout == 'bids': config.execution.fs_subjects_dir = output_dir / 'sourcedata' / 'freesurfer' elif output_layout == 'legacy': config.execution.fs_subjects_dir = output_dir / 'freesurfer' + if config.execution.nibabies_dir is None: if output_layout == 'bids': config.execution.nibabies_dir = output_dir elif output_layout == 'legacy': config.execution.nibabies_dir = output_dir / 'nibabies' + if config.workflow.surface_recon_method == 'mcribs': if output_layout == 'bids': config.execution.mcribs_dir = output_dir / 'sourcedata' / 'mcribs' @@ -909,7 +917,6 @@ def parse_args(args=None, namespace=None): participant_ids=config.execution.participant_label, session_ids=config.execution.session_id, ) - config.workflow.skull_strip_template = config.workflow.skull_strip_template[0] # finally, write config to file config_file = config.execution.work_dir / config.execution.run_uuid / 'config.toml' diff --git a/nibabies/cli/run.py b/nibabies/cli/run.py index fc1c8a36..de9a9c42 100755 --- a/nibabies/cli/run.py +++ b/nibabies/cli/run.py @@ -154,6 +154,7 @@ def main(): config.execution.bids_dir, config.execution.nibabies_dir, config.execution.dataset_links, + config.execution._config_hash, ) write_bidsignore(config.execution.nibabies_dir) diff --git a/nibabies/config.py b/nibabies/config.py index 23806f43..66524c39 100644 --- a/nibabies/config.py +++ b/nibabies/config.py @@ -433,6 +433,7 @@ class execution(_Config): write_graph = False """Write out the computational graph corresponding to the planned preprocessing.""" _layout = None + _config_hash = None _paths = ( 'anat_derivatives', diff --git a/nibabies/utils/bids.py b/nibabies/utils/bids.py index ff330330..8fc01aa9 100644 --- a/nibabies/utils/bids.py +++ b/nibabies/utils/bids.py @@ -40,7 +40,7 @@ def write_bidsignore(deriv_dir): ignore_file.write_text('\n'.join(bids_ignore) + '\n') -def write_derivative_description(bids_dir, deriv_dir, dataset_links=None): +def write_derivative_description(bids_dir, deriv_dir, dataset_links=None, config_hash=None): from nibabies import __version__ DOWNLOAD_URL = f'https://github.com/nipreps/nibabies/archive/{__version__}.tar.gz' @@ -56,6 +56,7 @@ def write_derivative_description(bids_dir, deriv_dir, dataset_links=None): 'Name': 'NiBabies', 'Version': __version__, 'CodeURL': DOWNLOAD_URL, + 'ConfigurationHash': config_hash, } ], 'HowToAcknowledge': 'TODO', From 2a4d8fe344b00fc42552356015a1d2ac6b643f14 Mon Sep 17 00:00:00 2001 From: mathiasg Date: Mon, 25 Aug 2025 15:40:42 -0400 Subject: [PATCH 05/14] enh: add hash to report file --- nibabies/cli/run.py | 3 +++ nibabies/reports/core.py | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/nibabies/cli/run.py b/nibabies/cli/run.py index de9a9c42..55aef7a6 100755 --- a/nibabies/cli/run.py +++ b/nibabies/cli/run.py @@ -144,11 +144,14 @@ def main(): finally: from ..reports.core import generate_reports + add_hash = config.execution.output_layout == 'multiverse' + # Generate reports phase generate_reports( config.execution.unique_labels, config.execution.nibabies_dir, config.execution.run_uuid, + config.execution._config_hash if add_hash else None, ) write_derivative_description( config.execution.bids_dir, diff --git a/nibabies/reports/core.py b/nibabies/reports/core.py index ed7ccdd2..545c417c 100644 --- a/nibabies/reports/core.py +++ b/nibabies/reports/core.py @@ -24,6 +24,7 @@ def run_reports( session=session, bootstrap_file=load_data.readable('reports-spec.yml'), reportlets_dir=reportlets_dir, + out_filename=out_filename, ).generate_report() @@ -33,6 +34,7 @@ def generate_reports( run_uuid, work_dir=None, packagename=None, + config_hash=None, ): """Execute run_reports on a list of subjects.""" reportlets_dir = None @@ -41,6 +43,14 @@ def generate_reports( report_errors = [] for subject, session in sub_ses_list: + # Determine the output filename + out_filename = f'sub-{subject}' + if session is not None: + out_filename += f'_ses-{session}' + if config_hash is not None: + out_filename += f'_{config_hash}' + out_filename += '.html' + report_errors.append( run_reports( output_dir, @@ -49,6 +59,7 @@ def generate_reports( session=session, packagename=packagename, reportlets_dir=reportlets_dir, + out_filename=out_filename, ) ) From c59c6eaa2431fdd76118e611802bb7d4c613d86f Mon Sep 17 00:00:00 2001 From: Mathias Goncalves Date: Wed, 27 Aug 2025 12:09:43 -0400 Subject: [PATCH 06/14] fix: multiverse options --- nibabies/cli/parser.py | 16 +++++++++++----- nibabies/workflows/base.py | 1 - nibabies/workflows/bold/outputs.py | 4 ++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/nibabies/cli/parser.py b/nibabies/cli/parser.py index 1217ad3a..26d448d3 100644 --- a/nibabies/cli/parser.py +++ b/nibabies/cli/parser.py @@ -843,23 +843,29 @@ def parse_args(args=None, namespace=None): version = config.environment.version output_layout = config.execution.output_layout config.execution._config_hash = config.hash_config(config.get()) - if output_layout == 'multiverse': - output_dir += f'-{version.split("+", 1)[0]}-{config.execution.config_hash}' if config.execution.fs_subjects_dir is None: - if output_layout == 'bids': + if output_layout in ('bids', 'multiverse'): config.execution.fs_subjects_dir = output_dir / 'sourcedata' / 'freesurfer' elif output_layout == 'legacy': config.execution.fs_subjects_dir = output_dir / 'freesurfer' if config.execution.nibabies_dir is None: - if output_layout == 'bids': + if output_layout in 'bids': config.execution.nibabies_dir = output_dir elif output_layout == 'legacy': config.execution.nibabies_dir = output_dir / 'nibabies' + elif output_layout == 'multiverse': + config.loggers.cli.warning( + 'Multiverse output selected - assigning output directory based on version' + ' and configuration hash.' + ) + config.execution.nibabies_dir = ( + output_dir / f'nibabies-{version.split("+", 1)[0]}-{config.execution._config_hash}' + ) if config.workflow.surface_recon_method == 'mcribs': - if output_layout == 'bids': + if output_layout in ('bids', 'multiverse'): config.execution.mcribs_dir = output_dir / 'sourcedata' / 'mcribs' elif output_layout == 'legacy': config.execution.mcribs_dir = output_dir / 'mcribs' diff --git a/nibabies/workflows/base.py b/nibabies/workflows/base.py index fea9c574..20ca92cc 100644 --- a/nibabies/workflows/base.py +++ b/nibabies/workflows/base.py @@ -368,7 +368,6 @@ def init_single_subject_wf( ) anat = reference_anat.lower() # To be used for workflow connections - LOGGER.info( 'Collected the following data for %s:\nRaw:\n%s\n\nDerivatives:\n\n%s\n', f'sub-{subject_id}' if not session_id else f'sub-{subject_id}_ses-{session_id}', diff --git a/nibabies/workflows/bold/outputs.py b/nibabies/workflows/bold/outputs.py index 24cdd4a5..8d051f02 100644 --- a/nibabies/workflows/bold/outputs.py +++ b/nibabies/workflows/bold/outputs.py @@ -32,7 +32,7 @@ from nibabies import config from nibabies._types import Anatomical -from nibabies.config import DEFAULT_DISMISS_ENTITIES, DEFAULT_MEMORY_MIN_GB, dismiss_echo +from nibabies.config import DEFAULT_DISMISS_ENTITIES, DEFAULT_MEMORY_MIN_GB, dismiss_entities from nibabies.interfaces import DerivativesDataSink from nibabies.interfaces.bids import BIDSURI @@ -487,7 +487,7 @@ def init_ds_registration_wf( mode='image', suffix='xfm', extension='.txt', - dismiss_entities=dismiss_echo(['part']), + dismiss_entities=dismiss_entities(['part']), **{'from': source, 'to': dest}, ), name='ds_xform', From 9d4a4fe780cab37f68bca36d3b64e1558be0ab55 Mon Sep 17 00:00:00 2001 From: Mathias Goncalves Date: Wed, 27 Aug 2025 17:09:33 -0400 Subject: [PATCH 07/14] enh: add hash to datasinks if multiverse --- nibabies/workflows/base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nibabies/workflows/base.py b/nibabies/workflows/base.py index 20ca92cc..cabcaf36 100644 --- a/nibabies/workflows/base.py +++ b/nibabies/workflows/base.py @@ -836,6 +836,10 @@ def clean_datasinks(workflow: pe.Workflow) -> pe.Workflow: for node in workflow.list_node_names(): if node.split('.')[-1].startswith('ds_'): workflow.get_node(node).interface.out_path_base = '' + workflow.get_node(node).interface.inputs.base_directory = config.execution.output_dir + + if config.execution.output_layout == 'multiverse': + workflow.get_node(node).interface.inputs.hash = config.execution.parameters_hash return workflow From cb471364b8428211ba7faedfeaa04e600049f014 Mon Sep 17 00:00:00 2001 From: Mathias Goncalves Date: Wed, 27 Aug 2025 22:11:32 -0400 Subject: [PATCH 08/14] fix: make config attribute public, use match statements --- nibabies/cli/parser.py | 64 +++++++++++++++++++++++++++--------------- nibabies/cli/run.py | 4 +-- nibabies/config.py | 3 +- 3 files changed, 45 insertions(+), 26 deletions(-) diff --git a/nibabies/cli/parser.py b/nibabies/cli/parser.py index 26d448d3..76691249 100644 --- a/nibabies/cli/parser.py +++ b/nibabies/cli/parser.py @@ -842,33 +842,51 @@ def parse_args(args=None, namespace=None): work_dir = config.execution.work_dir version = config.environment.version output_layout = config.execution.output_layout - config.execution._config_hash = config.hash_config(config.get()) - - if config.execution.fs_subjects_dir is None: - if output_layout in ('bids', 'multiverse'): - config.execution.fs_subjects_dir = output_dir / 'sourcedata' / 'freesurfer' - elif output_layout == 'legacy': - config.execution.fs_subjects_dir = output_dir / 'freesurfer' + config.execution.parameters_hash = config.hash_config(config.get()) + # Multiverse behaves as a cross between bids and legacy if config.execution.nibabies_dir is None: - if output_layout in 'bids': - config.execution.nibabies_dir = output_dir - elif output_layout == 'legacy': - config.execution.nibabies_dir = output_dir / 'nibabies' - elif output_layout == 'multiverse': - config.loggers.cli.warning( - 'Multiverse output selected - assigning output directory based on version' - ' and configuration hash.' - ) - config.execution.nibabies_dir = ( - output_dir / f'nibabies-{version.split("+", 1)[0]}-{config.execution._config_hash}' - ) + match output_layout: + case 'bids': + config.execution.nibabies_dir = output_dir + case 'legacy': + config.execution.nibabies_dir = output_dir / 'nibabies' + case 'multiverse': + config.loggers.cli.warning( + 'Multiverse output selected - assigning output directory based on version' + ' and configuration hash.' + ) + config.execution.nibabies_dir = ( + output_dir + / f'nibabies-{version.split("+", 1)[0]}-{config.execution.parameters_hash}' + ) + case _: + config.loggers.cli.warning('Unknown output layout %s', output_layout) + pass + + nibabies_dir = config.execution.nibabies_dir + + if config.execution.fs_subjects_dir is None: + match output_layout: + case 'bids': + config.execution.fs_subjects_dir = output_dir / 'sourcedata' / 'freesurfer' + case 'legacy': + config.execution.fs_subjects_dir = output_dir / 'freesurfer' + case 'multiverse': + config.execution.fs_subjects_dir = nibabies_dir / 'sourcedata' / 'freesurfer' + case _: + pass if config.workflow.surface_recon_method == 'mcribs': - if output_layout in ('bids', 'multiverse'): - config.execution.mcribs_dir = output_dir / 'sourcedata' / 'mcribs' - elif output_layout == 'legacy': - config.execution.mcribs_dir = output_dir / 'mcribs' + match output_layout: + case 'bids': + config.execution.mcribs_dir = output_dir / 'sourcedata' / 'mcribs' + case 'legacy': + config.execution.mcribs_dir = output_dir / 'mcribs' + case 'multiverse': + config.execution.mcribs_dir = nibabies_dir / 'sourcedata' / 'mcribs' + case _: + pass # Ensure the directory is created config.execution.mcribs_dir.mkdir(exist_ok=True, parents=True) diff --git a/nibabies/cli/run.py b/nibabies/cli/run.py index 55aef7a6..b577b13f 100755 --- a/nibabies/cli/run.py +++ b/nibabies/cli/run.py @@ -151,13 +151,13 @@ def main(): config.execution.unique_labels, config.execution.nibabies_dir, config.execution.run_uuid, - config.execution._config_hash if add_hash else None, + config_hash=config.execution.parameters_hash if add_hash else None, ) write_derivative_description( config.execution.bids_dir, config.execution.nibabies_dir, config.execution.dataset_links, - config.execution._config_hash, + config.execution.parameters_hash, ) write_bidsignore(config.execution.nibabies_dir) diff --git a/nibabies/config.py b/nibabies/config.py index 66524c39..c4a4c91e 100644 --- a/nibabies/config.py +++ b/nibabies/config.py @@ -410,6 +410,8 @@ class execution(_Config): output_spaces = None """List of (non)standard spaces designated (with the ``--output-spaces`` flag of the command line) as spatial references for outputs.""" + parameters_hash = None + """Unique hash of the current configuration parameters.""" reference_anat = None """Force usage of this anatomical scan as the structural reference.""" reports_only = False @@ -433,7 +435,6 @@ class execution(_Config): write_graph = False """Write out the computational graph corresponding to the planned preprocessing.""" _layout = None - _config_hash = None _paths = ( 'anat_derivatives', From d0ee4df83d6f66a336ec54115e514472abbc01a3 Mon Sep 17 00:00:00 2001 From: Mathias Goncalves Date: Wed, 27 Aug 2025 22:13:24 -0400 Subject: [PATCH 09/14] rf: update reporting, add config hash parameter --- nibabies/reports/core.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/nibabies/reports/core.py b/nibabies/reports/core.py index 545c417c..9331b64f 100644 --- a/nibabies/reports/core.py +++ b/nibabies/reports/core.py @@ -10,9 +10,9 @@ def run_reports( subject, run_uuid, session=None, - out_filename=None, + bootstrap_file=None, + out_filename='report.html', reportlets_dir=None, - packagename=None, ): """ Run the reports. @@ -22,7 +22,7 @@ def run_reports( run_uuid, subject=subject, session=session, - bootstrap_file=load_data.readable('reports-spec.yml'), + bootstrap_file=load_data('reports-spec.yml'), reportlets_dir=reportlets_dir, out_filename=out_filename, ).generate_report() @@ -32,8 +32,9 @@ def generate_reports( sub_ses_list, output_dir, run_uuid, + *, work_dir=None, - packagename=None, + bootstrap_file=None, config_hash=None, ): """Execute run_reports on a list of subjects.""" @@ -44,22 +45,22 @@ def generate_reports( report_errors = [] for subject, session in sub_ses_list: # Determine the output filename - out_filename = f'sub-{subject}' + html_report = f'sub-{subject}' if session is not None: - out_filename += f'_ses-{session}' + html_report += f'_ses-{session}' if config_hash is not None: - out_filename += f'_{config_hash}' - out_filename += '.html' + html_report += f'_{config_hash}' + html_report += '.html' report_errors.append( run_reports( output_dir, subject, run_uuid, - session=session, - packagename=packagename, + bootstrap_file=bootstrap_file, reportlets_dir=reportlets_dir, - out_filename=out_filename, + out_filename=html_report, + session=session, ) ) From 8f26bfb599422bdfb152ace73f5c3199b3bc4649 Mon Sep 17 00:00:00 2001 From: Mathias Goncalves Date: Wed, 27 Aug 2025 22:13:43 -0400 Subject: [PATCH 10/14] fix: output to nibabies dir --- nibabies/workflows/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nibabies/workflows/base.py b/nibabies/workflows/base.py index cabcaf36..38951bc0 100644 --- a/nibabies/workflows/base.py +++ b/nibabies/workflows/base.py @@ -836,7 +836,7 @@ def clean_datasinks(workflow: pe.Workflow) -> pe.Workflow: for node in workflow.list_node_names(): if node.split('.')[-1].startswith('ds_'): workflow.get_node(node).interface.out_path_base = '' - workflow.get_node(node).interface.inputs.base_directory = config.execution.output_dir + workflow.get_node(node).interface.inputs.base_directory = config.execution.nibabies_dir if config.execution.output_layout == 'multiverse': workflow.get_node(node).interface.inputs.hash = config.execution.parameters_hash From e228bb34d28409e335e3755c4bfb637e9dbf1943 Mon Sep 17 00:00:00 2001 From: Mathias Goncalves Date: Thu, 28 Aug 2025 11:39:35 -0400 Subject: [PATCH 11/14] enh: add hash when copying derivatives --- nibabies/utils/derivatives.py | 16 ++++++++++++++-- nibabies/workflows/base.py | 6 ++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/nibabies/utils/derivatives.py b/nibabies/utils/derivatives.py index 6191a1d5..aae8e745 100644 --- a/nibabies/utils/derivatives.py +++ b/nibabies/utils/derivatives.py @@ -136,6 +136,7 @@ def copy_derivatives( modality: str, subject_id: str, session_id: str | None = None, + config_hash: str | None = None, ) -> None: """ Creates a copy of any found derivatives into output directory. @@ -154,8 +155,19 @@ def copy_derivatives( if not isinstance(deriv, str): continue deriv = Path(deriv) - - shutil.copy2(deriv, outpath / deriv.name) + outname = deriv.name + + if config_hash: + ents = outname.split('_') + if any(ent.startswith('hash-') for ent in ents): + # Avoid adding another hash + pass + else: + idx = 2 if ents[1].startswith('ses-') else 1 + ents.insert(idx, f'hash-{config_hash}') + outname = '_'.join(ents) + + shutil.copy2(deriv, outpath / outname) json = deriv.parent / (deriv.name.split('.')[0] + '.json') if json.exists(): shutil.copy2(json, outpath / json.name) diff --git a/nibabies/workflows/base.py b/nibabies/workflows/base.py index 38951bc0..f62f53ce 100644 --- a/nibabies/workflows/base.py +++ b/nibabies/workflows/base.py @@ -330,6 +330,9 @@ def init_single_subject_wf( modality='anat', subject_id=f'sub-{subject_id}', session_id=f'ses-{session_id}' if session_id else None, + config_hash=config.execution.parameters_hash + if config.execution.output_layout == 'multiverse' + else None, ) # Determine some session level options here, as we should have @@ -738,6 +741,9 @@ def init_single_subject_wf( modality='func', subject_id=f'sub-{subject_id}', session_id=f'ses-{session_id}' if session_id else None, + config_hash=config.execution.parameters_hash + if config.execution.output_layout == 'multiverse' + else None, ) bold_wf = init_bold_wf( From 4ccbbd93162bbfee6e9dfd2efe1860f243bad33c Mon Sep 17 00:00:00 2001 From: Mathias Goncalves Date: Fri, 5 Sep 2025 09:45:38 -0400 Subject: [PATCH 12/14] fix: add hash to copied metadata derivatives as well --- nibabies/utils/derivatives.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nibabies/utils/derivatives.py b/nibabies/utils/derivatives.py index aae8e745..d8c395df 100644 --- a/nibabies/utils/derivatives.py +++ b/nibabies/utils/derivatives.py @@ -168,6 +168,6 @@ def copy_derivatives( outname = '_'.join(ents) shutil.copy2(deriv, outpath / outname) - json = deriv.parent / (deriv.name.split('.')[0] + '.json') + json = deriv.parent / (outname.split('.')[0] + '.json') if json.exists(): shutil.copy2(json, outpath / json.name) From fdedcc44844a5d7031319497cd5e887de2eee815 Mon Sep 17 00:00:00 2001 From: Mathias Goncalves Date: Tue, 9 Sep 2025 09:42:12 -0400 Subject: [PATCH 13/14] pin: dev niworkflows --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 03295de6..5414fe0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,8 @@ dependencies = [ "nireports >= 23.2.0", "nitime", "nitransforms >= 24.1.1", - "niworkflows >= 1.13.1", + #"niworkflows >= 1.13.1", + "niworkflows @ git+https://github.com/nipreps/niworkflows.git@master", "numpy >= 1.21.0", "packaging", "pandas < 3", From e18406b5c1004968516f0b0d9baf823a2285412d Mon Sep 17 00:00:00 2001 From: mathiasg Date: Wed, 10 Sep 2025 11:24:37 -0400 Subject: [PATCH 14/14] fix: append hash to fs/mcribs directories --- nibabies/cli/parser.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nibabies/cli/parser.py b/nibabies/cli/parser.py index 76691249..56c865eb 100644 --- a/nibabies/cli/parser.py +++ b/nibabies/cli/parser.py @@ -873,7 +873,9 @@ def parse_args(args=None, namespace=None): case 'legacy': config.execution.fs_subjects_dir = output_dir / 'freesurfer' case 'multiverse': - config.execution.fs_subjects_dir = nibabies_dir / 'sourcedata' / 'freesurfer' + config.execution.fs_subjects_dir = ( + nibabies_dir / 'sourcedata' / f'freesurfer-{config.execution.parameters_hash}' + ) case _: pass @@ -884,7 +886,9 @@ def parse_args(args=None, namespace=None): case 'legacy': config.execution.mcribs_dir = output_dir / 'mcribs' case 'multiverse': - config.execution.mcribs_dir = nibabies_dir / 'sourcedata' / 'mcribs' + config.execution.mcribs_dir = ( + nibabies_dir / 'sourcedata' / f'mcribs-{config.execution.parameters_hash}' + ) case _: pass # Ensure the directory is created