Merge pull request #13 from neurorepro/adds_populate_intended_for

Add matching by custom label
Thank you, @neurorepro

Author: pvelasco

.gitignore

@@ -6,3 +6,5 @@
 venvs/
 _build/
 build/
+.vscode/
+

docs/heuristics.rst

@@ -117,8 +117,16 @@ The parameters that can be specified and the allowed options are defined in ``bi
    * ``'ImagingVolume'``: both ``fmaps`` and images will need to have the same the imaging
      volume (the header affine transformation: position, orientation and voxel size, as well
      as number of voxels along each dimensions).
-   * ``'AcquisitionLabel'``: it checks for what modality (``anat``, ``func``, ``dwi``) each
-     ``fmap`` is intended by checking the ``_acq-`` label in the filename
+   * ``'ModalityAcquisitionLabel'``: it checks for what modality (``anat``, ``func``, ``dwi``) each
+     ``fmap`` is intended by checking the ``_acq-`` label in the ``fmap`` filename and finding
+     corresponding modalities (e.g. ``_acq-fmri``, ``_acq-bold`` and ``_acq-func`` will be matched
+     with the ``func`` modality)
+   * ``'CustomAcquisitionLabel'``: it checks for what modality images each  ``fmap`` is intended
+     by checking the ``_acq-`` custom label (e.g. ``_acq-XYZ42``) in the ``fmap`` filename, and
+     matching it with:
+     - the corresponding modality image ``_acq-`` label for modalities other than ``func``
+     (e.g. ``_acq-XYZ42`` for ``dwi`` images)
+     - the corresponding image ``_task-`` label for the ``func`` modality (e.g. ``_task-XYZ42``)
    * ``'Force'``: forces ``heudiconv`` to consider any ``fmaps`` in the session to be
      suitable for any image, no matter what the imaging parameters are.
 

heudiconv/bids.py

@@ -61,7 +61,8 @@ class BIDSError(Exception):
 AllowedFmapParameterMatching = [
     'Shims',
     'ImagingVolume',
-    'AcquisitionLabel',
+    'ModalityAcquisitionLabel',
+    'CustomAcquisitionLabel',
     'Force',
 ]
 # Key info returned by get_key_info_for_fmap_assignment when
@@ -625,7 +626,7 @@ def get_key_info_for_fmap_assignment(json_file, matching_parameter):
         nifti_file = nifti_file[0] 
         nifti_header = nb_load(nifti_file).header
         key_info = [nifti_header.get_best_affine(), nifti_header.get_data_shape()[:3]]
-    elif matching_parameter == 'AcquisitionLabel':
+    elif matching_parameter == 'ModalityAcquisitionLabel':
         # Check the acq label for the fmap and the modality for others:
         modality = op.basename(op.dirname(json_file))
         if modality == 'fmap':
@@ -639,6 +640,16 @@ def get_key_info_for_fmap_assignment(json_file, matching_parameter):
                 key_info = ['anat']
         else:
             key_info = [modality]
+    elif matching_parameter == 'CustomAcquisitionLabel':
+        modality = op.basename(op.dirname(json_file))
+        if modality  == 'func':
+            # extract the <task> entity:
+            custom_label = BIDSFile.parse(op.basename(json_file))['task']
+        else:
+            # extract the <acq> entity:
+            custom_label = BIDSFile.parse(op.basename(json_file))['acq']
+        # Get the custom acquisition label, acq_label is None if no custom field found
+        key_info = [custom_label]
     elif matching_parameter == 'Force':
         # We want to force the matching, so just return some string
         # regardless of the image
@@ -649,7 +660,6 @@ def get_key_info_for_fmap_assignment(json_file, matching_parameter):
 
     return key_info
 
-
 def find_compatible_fmaps_for_run(json_file, fmap_groups, matching_parameters):
     """
     Finds compatible fmaps for a given run, for populate_intended_for.
@@ -685,10 +695,14 @@ def find_compatible_fmaps_for_run(json_file, fmap_groups, matching_parameters):
         # the fmaps in the group:
         compatible = False
         for param in matching_parameters:
+            json_info_1st_item = json_info[param][0]
             fm_info = get_key_info_for_fmap_assignment(fm_group[0], param)
             # for the case in which key_info is a list of strings:
-            if isinstance(json_info[param][0], str):
+            if isinstance(json_info_1st_item, str):
                 compatible = json_info[param] == fm_info
+            # for the case when no key info was found (e.g. "acq" field does not exist)
+            elif json_info_1st_item is None:
+                compatible = False
             else:
                 # allow for tiny differences between the affines etc
                 compatible = all(np.allclose(x, y) for x, y in zip(json_info[param], fm_info))

heudiconv/tests/test_bids.py

@@ -4,8 +4,11 @@
 import re
 import os
 import os.path as op
+from pathlib import Path
 from random import (random,
                     shuffle,
+                    choice,
+                    seed
                     )
 from datetime import (datetime,
                       timedelta,
@@ -15,6 +18,7 @@
 from glob import glob
 
 import nibabel
+import string
 from numpy import testing as np_testing
 
 from heudiconv.utils import (
@@ -46,6 +50,15 @@
 
 import pytest
 
+def gen_rand_label(label_size, label_seed, seed_stdout=True):
+    seed(label_seed)
+    rand_char = ''.join(choice(string.ascii_letters) for _ in range(label_size-1))
+    seed(label_seed)
+    rand_num = choice(string.digits)
+    if seed_stdout:
+        print(f'Seed used to generate custom label: {label_seed}')
+    return rand_char + rand_num
+
 def test_maybe_na():
     for na in '', ' ', None, 'n/a', 'N/A', 'NA':
         assert maybe_na(na) == 'n/a'
@@ -67,7 +80,7 @@ def test_treat_age():
 
 SHIM_LENGTH = 6
 TODAY = datetime.today()
-
+LABEL_SEED = int.from_bytes(os.urandom(8), byteorder="big")
 
 A_SHIM = [random() for i in range(SHIM_LENGTH)]
 def test_get_shim_setting(tmpdir):
@@ -86,9 +99,13 @@ def test_get_shim_setting(tmpdir):
     assert get_shim_setting(json_name) == A_SHIM
 
 
-def test_get_key_info_for_fmap_assignment(tmpdir):
+def test_get_key_info_for_fmap_assignment(tmpdir, label_size=4, label_seed=LABEL_SEED):
     """
-    Test get_key_info_for_fmap_assignment
+    Test get_key_info_for_fmap_assignment.
+    
+    label_size and label_seed are used for the "CustomAcquisitionLabel" matching
+    parameter. label_size is the size of the random label while label_seed is 
+    the seed for the random label creation.
     """
 
     nifti_file = op.join(TESTS_DATA_PATH, 'sample_nifti.nii.gz')
@@ -123,9 +140,9 @@ def test_get_key_info_for_fmap_assignment(tmpdir):
     )
     assert key_info == [KeyInfoForForce]
 
-    # 5) matching_parameter = 'AcquisitionLabel'
+    # 5) matching_parameter = 'ModalityAcquisitionLabel'
     for d in ['fmap', 'func', 'dwi', 'anat']:
-        os.makedirs(op.join(str(tmpdir), d))
+        Path(op.join(str(tmpdir), d)).mkdir(parents=True, exist_ok=True)
     for (dirname, fname, expected_key_info) in [
         ('fmap', 'sub-foo_acq-fmri_epi.json', 'func'),
         ('fmap', 'sub-foo_acq-bold_epi.json', 'func'),
@@ -140,7 +157,24 @@ def test_get_key_info_for_fmap_assignment(tmpdir):
         json_name = op.join(str(tmpdir), dirname, fname)
         save_json(json_name, {SHIM_KEY: A_SHIM})
         assert [expected_key_info] == get_key_info_for_fmap_assignment(
-            json_name, matching_parameter='AcquisitionLabel'
+            json_name, matching_parameter='ModalityAcquisitionLabel'
+        )
+
+    # 6) matching_parameter = 'CustomAcquisitionLabel'
+    A_LABEL = gen_rand_label(label_size, label_seed)
+    for d in ['fmap', 'func', 'dwi', 'anat']:
+        Path(op.join(str(tmpdir), d)).mkdir(parents=True, exist_ok=True)
+        
+    for (dirname, fname, expected_key_info) in [
+        ('fmap', f'sub-foo_acq-{A_LABEL}_epi.json', A_LABEL),
+        ('func', f'sub-foo_task-{A_LABEL}_acq-foo_bold.json', A_LABEL),
+        ('dwi', f'sub-foo_acq-{A_LABEL}_dwi.json', A_LABEL),
+        ('anat', f'sub-foo_acq-{A_LABEL}_T1w.json', A_LABEL),
+    ]:
+        json_name = op.join(str(tmpdir), dirname, fname)
+        save_json(json_name, {SHIM_KEY: A_SHIM})
+        assert [expected_key_info] == get_key_info_for_fmap_assignment(
+            json_name, matching_parameter='CustomAcquisitionLabel'
         )
 
     # Finally: invalid matching_parameters:
@@ -500,6 +534,158 @@ def create_dummy_no_shim_settings_bids_session(session_path):
 
     return session_struct, expected_result, expected_fmap_groups, expected_compatible_fmaps
 
+def create_dummy_no_shim_settings_custom_label_bids_session(session_path, label_size=4, label_seed=LABEL_SEED):
+    """
+    Creates a dummy BIDS session, with slim json files and empty nii.gz
+    The fmap files are pepolar
+    The json files don't have ShimSettings
+    The fmap files have a custom ACQ label matching:
+        - TASK label for <func> modality
+        - ACQ label for any other modality (e.g. <dwi>)
+
+    Parameters:
+    ----------
+    session_path : str or os.path
+        path to the session (or subject) level folder
+    label_size : int, optional
+        size of the random label
+    label_seed : int, optional
+        seed for the random label creation
+
+    Returns:
+    -------
+    session_struct : dict
+        Structure of the directory that was created
+    expected_result : dict
+        dictionary with fmap names as keys and the expected "IntendedFor" as
+        values.
+    None
+        it returns a third argument (None) to have the same signature as
+        create_dummy_pepolar_bids_session
+    """
+    session_parent, session_basename = op.split(session_path.rstrip(op.sep))
+    if session_basename.startswith('ses-'):
+        prefix = op.split(session_parent)[1] + '_' + session_basename
+    else:
+        prefix = session_basename
+
+    # 1) Simulate the file structure for a session:
+
+    # Dict with the file structure for the session.
+    # All json files will be empty.
+    # -anat:
+    anat_struct = {
+        f'{prefix}_{mod}.{ext}': dummy_content
+        for ext, dummy_content in zip(['nii.gz', 'json'], ['', {}])
+        for mod in ['T1w', 'T2w']
+    }
+    # -dwi:
+    label_seed += 1
+    DWI_LABEL = gen_rand_label(label_size, label_seed)
+    dwi_struct = {
+        f'{prefix}_acq-{DWI_LABEL}_run-{runNo}_dwi.{ext}': dummy_content
+        for ext, dummy_content in zip(['nii.gz', 'json'], ['', {}])
+        for runNo in [1, 2]
+    }
+    # -func:
+    label_seed += 1
+    FUNC_LABEL = gen_rand_label(label_size, label_seed)
+    func_struct = {
+        f'{prefix}_task-{FUNC_LABEL}_acq-{acq}_bold.{ext}': dummy_content
+        for ext, dummy_content in zip(['nii.gz', 'json'], ['', {}])
+        for acq in ['A', 'B']
+    }
+    # -fmap:
+    fmap_struct = {
+        f'{prefix}_acq-{acq}_dir-{d}_run-{r}_epi.{ext}': dummy_content
+        for ext, dummy_content in zip(['nii.gz', 'json'], ['', {}])
+        for acq in [DWI_LABEL, FUNC_LABEL]
+        for d in ['AP', 'PA']
+        for r in [1, 2]
+    }
+    expected_fmap_groups = {
+        f'{prefix}_acq-{acq}_run-{r}_epi': [
+            f'{op.join(session_path, "fmap", prefix)}_acq-{acq}_dir-{d}_run-{r}_epi.json'
+            for d in ['AP', 'PA']
+        ]
+        for acq in [DWI_LABEL, FUNC_LABEL]
+        for r in [1, 2]
+    }
+
+    # structure for the full session (init the OrderedDict as a list to preserve order):
+    session_struct = OrderedDict([
+        ('fmap', fmap_struct),
+        ('anat', anat_struct),
+        ('dwi', dwi_struct),
+        ('func', func_struct),
+    ])
+    # add "_scans.tsv" file to the session_struct
+    scans_file_content = generate_scans_tsv(session_struct)
+    session_struct.update({'{p}_scans.tsv'.format(p=prefix): scans_file_content})
+
+    create_tree(session_path, session_struct)
+
+    # 2) Now, let's create a dict with the fmap groups compatible for each run
+    # -anat: empty
+    expected_compatible_fmaps = {
+        f'{op.join(session_path, "anat", prefix)}_{mod}.json': {}
+        for mod in ['T1w', 'T2w']
+    }
+    # -dwi: each of the runs (1, 2) is compatible with both of the dwi fmaps (1, 2):
+    expected_compatible_fmaps.update({
+        f'{op.join(session_path, "dwi", prefix)}_acq-{DWI_LABEL}_run-{runNo}_dwi.json': {
+            key: val for key, val in expected_fmap_groups.items() if key in [
+                f'{prefix}_acq-{DWI_LABEL}_run-{r}_epi' for r in [1, 2]
+            ]
+        }
+        for runNo in [1, 2]
+    })
+    # -func: each of the acq (A, B) is compatible w/ both fmap fMRI runs (1, 2)
+    expected_compatible_fmaps.update({
+        f'{op.join(session_path, "func", prefix)}_task-{FUNC_LABEL}_acq-{acq}_bold.json': {
+            key: val for key, val in expected_fmap_groups.items() if key in [
+                f'{prefix}_acq-{FUNC_LABEL}_run-{r}_epi' for r in [1, 2]
+           ]
+        }
+        for acq in ['A', 'B']
+    })
+
+    # 3) Now, let's create a dict with what we expect for the "IntendedFor":
+    # NOTE: The "expected_prefix" (the beginning of the path to the
+    # "IntendedFor") should be relative to the subject level (see:
+    # https://bids-specification.readthedocs.io/en/stable/04-modality-specific-files/01-magnetic-resonance-imaging-data.html#fieldmap-data)
+
+    sub_match = re.findall('(sub-([a-zA-Z0-9]*))', session_path)
+    sub_str = sub_match[0][0]
+    expected_prefix = session_path.split(sub_str)[-1].split(op.sep)[-1]
+
+    # dict, with fmap names as keys and the expected "IntendedFor" as values.
+    expected_result = {
+        # (runNo=1 goes with the long list, runNo=2 goes with None):
+        f'{prefix}_acq-{DWI_LABEL}_dir-{d}_run-{runNo}_epi.json': intended_for
+        for runNo, intended_for in zip(
+            [1, 2],
+            [[op.join(expected_prefix, 'dwi', f'{prefix}_acq-{DWI_LABEL}_run-{r}_dwi.nii.gz') for r in [1,2]],
+             None]
+        )
+        for d in ['AP', 'PA']
+    }
+    expected_result.update(
+        {
+            # The first "fMRI" run gets all files in the "func" folder;
+            # the second shouldn't get any.
+            f'{prefix}_acq-{FUNC_LABEL}_dir-{d}_run-{runNo}_epi.json': intended_for
+            for runNo, intended_for in zip(
+                [1, 2],
+                [[op.join(expected_prefix, 'func', f'{prefix}_task-{FUNC_LABEL}_acq-{acq}_bold.nii.gz')
+                  for acq in ['A', 'B']],
+                  None]
+            )
+            for d in ['AP', 'PA']
+        }
+    )
+
+    return session_struct, expected_result, expected_fmap_groups, expected_compatible_fmaps
 
 def create_dummy_magnitude_phase_bids_session(session_path):
     """
@@ -685,7 +871,8 @@ def test_find_fmap_groups(tmpdir, simulation_function):
 @pytest.mark.parametrize(
     "simulation_function, match_param", [
         (create_dummy_pepolar_bids_session, 'Shims'),
-        (create_dummy_no_shim_settings_bids_session, 'AcquisitionLabel'),
+        (create_dummy_no_shim_settings_bids_session, 'ModalityAcquisitionLabel'),
+        (create_dummy_no_shim_settings_custom_label_bids_session, 'CustomAcquisitionLabel'),
         (create_dummy_magnitude_phase_bids_session, 'Shims')
     ]
 )
@@ -726,7 +913,8 @@ def test_find_compatible_fmaps_for_run(tmpdir, simulation_function, match_param)
         for folder, expected_prefix in zip(['no_sessions/sub-1', 'sessions/sub-1/ses-pre'], ['', 'ses-pre'])
         for sim_func, mp in [
             (create_dummy_pepolar_bids_session, 'Shims'),
-            (create_dummy_no_shim_settings_bids_session, 'AcquisitionLabel'),
+            (create_dummy_no_shim_settings_bids_session, 'ModalityAcquisitionLabel'),
+            (create_dummy_no_shim_settings_custom_label_bids_session, 'CustomAcquisitionLabel'),
             (create_dummy_magnitude_phase_bids_session, 'Shims')
         ]
     ]
@@ -816,7 +1004,8 @@ def test_select_fmap_from_compatible_groups(tmpdir, folder, expected_prefix, sim
         for folder, expected_prefix in zip(['no_sessions/sub-1', 'sessions/sub-1/ses-pre'], ['', 'ses-pre'])
         for sim_func, mp in [
             (create_dummy_pepolar_bids_session, 'Shims'),
-            (create_dummy_no_shim_settings_bids_session, 'AcquisitionLabel'),
+            (create_dummy_no_shim_settings_bids_session, 'ModalityAcquisitionLabel'),
+            (create_dummy_no_shim_settings_custom_label_bids_session, 'CustomAcquisitionLabel'),
             (create_dummy_magnitude_phase_bids_session, 'Shims')
         ]
     ]