Merge pull request #21 from nimh-dsst/eric-testing

BIDS App preparation and serial/parallel behavior

Author: Arshitha

.gitignore

@@ -1,12 +1,9 @@
 __pycache__/
-as_qc_failures.csv
-*.swarm
 *_testing/
 *_examples/
-scripts_outputs/
 .idea/
+.vscode/
 examples/visualqc_prep/
 examples/sub-01/
-user-testing/*
-
-src/isolate_fsleyes_render_issue.py
+lookup.json
+rename.py

README.md

@@ -1,3 +1,7 @@
+import gzip
+import re
+import shutil
+import os
 import subprocess
 from os import fspath
 from pathlib import Path
@@ -51,6 +55,79 @@ def get_anat_dir_paths(subj_dir_path):
     return anat_dirs
 
 
+def compress_to_gz(input_file, output_file):
+    if not output_file.exists():
+        with open(input_file, 'rb') as f_input:
+            with gzip.open(output_file, 'wb') as f_output:
+                f_output.writelines(f_input)
+
+
+def copy_over_sidecar(scan_filepath, input_anat_dir, output_anat_dir):
+    prefix = '_'.join([i for i in re.split('_|\.', scan_filepath.name) if i not in ['defaced', 'nii', 'gz']])
+    filename = prefix + '.json'
+    json_sidecar = input_anat_dir / filename
+    shutil.copy2(json_sidecar, output_anat_dir / filename)
+
+
+def vqcdeface_prep(bids_input_dir, defaced_anat_dir, bids_defaced_outdir):
+    defacing_qc_dir = bids_defaced_outdir.parent / 'QC_prep' / 'defacing_QC'
+    interested_files = [f for f in defaced_anat_dir.rglob('*.nii.gz') if
+                        'work_dir' not in str(f).split('/')]
+    print(interested_files)
+    for defaced_img in interested_files:
+        entities = defaced_img.name.split('.')[0].split('_')
+        vqcd_subj_dir = defacing_qc_dir / f"{'/'.join(entities)}"
+        vqcd_subj_dir.mkdir(parents=True, exist_ok=True)
+
+        defaced_link = vqcd_subj_dir / 'defaced.nii.gz'
+        if not defaced_link.is_symlink():
+            defaced_link.symlink_to(defaced_img)
+        print(list(bids_input_dir.rglob(defaced_img.name)))
+        img = list(bids_input_dir.rglob(defaced_img.name))[0]
+        img_link = vqcd_subj_dir / 'orig.nii.gz'
+        if not img_link.is_symlink(): img_link.symlink_to(img)
+
+
+def reorganize_into_bids(input_bids_dir, subj_dir, sess_dir, primary_t1, bids_defaced_outdir, no_clean):
+    subj_id = subj_dir.name
+    sess_id = sess_dir.name if sess_dir else None
+
+    if sess_id:
+        anat_dirs = list(bids_defaced_outdir.joinpath(subj_id, sess_id).rglob('anat'))
+    else:
+        anat_dirs = list(bids_defaced_outdir.joinpath(subj_id).rglob('anat'))
+    # make workdir for each session within anat dir
+    for anat_dir in anat_dirs:
+        # iterate over all nii files within an anat dir to rename all primary and "other" scans
+        for nii_filepath in anat_dir.rglob('*nii*'):
+            if nii_filepath.name.startswith('tmp.99.result'):
+                # convert to nii.gz, rename and copy over to anat dir
+                gz_file = anat_dir / Path(primary_t1).name
+                compress_to_gz(nii_filepath, gz_file)
+
+                # copy over corresponding json sidecar
+                copy_over_sidecar(Path(primary_t1), input_bids_dir / anat_dir.relative_to(bids_defaced_outdir),
+                                  anat_dir)
+
+            elif nii_filepath.name.endswith('_defaced.nii.gz'):
+                new_filename = '_'.join(nii_filepath.name.split('_')[:-1]) + '.nii.gz'
+                shutil.copy2(nii_filepath, str(anat_dir / new_filename))
+
+                copy_over_sidecar(nii_filepath, input_bids_dir / anat_dir.relative_to(bids_defaced_outdir), anat_dir)
+
+        # move QC images and afni intermediate files to a new directory
+        intermediate_files_dir = anat_dir / 'work_dir'
+        intermediate_files_dir.mkdir(parents=True, exist_ok=True)
+        for dirpath in anat_dir.glob('*'):
+            if dirpath.name.startswith('workdir') or dirpath.name.endswith('QC'):
+                shutil.move(str(dirpath), str(intermediate_files_dir))
+
+        vqcdeface_prep(input_bids_dir, anat_dir, bids_defaced_outdir)
+
+        if not no_clean:
+            shutil.rmtree(intermediate_files_dir)
+
+
 def run_afni_refacer(primary_t1, others, subj_input_dir, sess_dir, output_dir):
     # constructing afni refacer command
     if primary_t1:
@@ -77,7 +154,7 @@ def run_afni_refacer(primary_t1, others, subj_input_dir, sess_dir, output_dir):
         refacer_cmd = f"@afni_refacer_run -input {primary_t1} -mode_deface -no_clean -prefix {fspath(subj_outdir / prefix)}"
 
         # TODO remove module load afni
-        full_cmd = f"module load afni ; {refacer_cmd}"
+        full_cmd = f"module load afni ; export OMP_NUM_THREADS=1 ; {refacer_cmd}"
 
         # TODO make log text less ugly; perhaps in a separate function
         log_filename = subj_outdir / 'defacing_pipeline.log'
@@ -102,6 +179,7 @@ def run_afni_refacer(primary_t1, others, subj_input_dir, sess_dir, output_dir):
 
             # register other scans to the primary scan
             register.register_to_primary_scan(subj_input_dir, new_afni_workdir, primary_t1, others, log_fileobj)
+
         else:
             log_fileobj.write(
                 f"@afni_refacer_run work directory not found. Most probably because the refacer command failed.")
@@ -110,20 +188,26 @@ def run_afni_refacer(primary_t1, others, subj_input_dir, sess_dir, output_dir):
         return missing_refacer_out
 
 
-def deface_primary_scan(subj_input_dir, sess_dir, mapping_dict, output_dir):
+def deface_primary_scan(input_bids_dir, subj_input_dir, sess_dir, mapping_dict, output_dir, no_clean):
     missing_refacer_outputs = []  # list to capture missing afni refacer workdirs
 
-    subj_id = subj_input_dir.name
-    sess_id = sess_dir.name if sess_dir else None
+    subj_id = os.path.basename(subj_input_dir)
+    sess_id = os.path.basename(sess_dir) if sess_dir else None
 
     if sess_dir:
         primary_t1 = mapping_dict[subj_id][sess_id]['primary_t1']
         others = [str(s) for s in mapping_dict[subj_id][sess_id]['others'] if s != primary_t1]
         missing_refacer_outputs.append(run_afni_refacer(primary_t1, others, subj_input_dir, sess_dir, output_dir))
+        print(f"Reorganizing {sess_dir} with defaced images into BIDS tree...\n")
+
     else:
         primary_t1 = mapping_dict[subj_id]['primary_t1']
         others = [str(s) for s in mapping_dict[subj_id]['others'] if s != primary_t1]
         missing_refacer_outputs.append(run_afni_refacer(primary_t1, others, subj_input_dir, "", output_dir))
+        print(f"Reorganizing {subj_input_dir} with defaced images into BIDS tree...\n")
+
+    # reorganizing the directory with defaced images into BIDS tree
+    reorganize_into_bids(input_bids_dir, subj_input_dir, sess_dir, primary_t1, output_dir, no_clean)
 
     return missing_refacer_outputs