Merge pull request #62 from ArcanaFramework/full-mounts

Full mount list for copy mode requirements

Author: tclose

fileformats/core/fileset.py

@@ -1305,7 +1305,10 @@ def copy(
         # Rule out any copy modes that are not supported given the collation mode
         # and file-system mounts the paths and destination directory reside on
         constraints = []
-        if FsMountIdentifier.on_cifs(dest_dir) and mode & self.CopyMode.symlink:
+        if (
+            not FsMountIdentifier.symlinks_supported(dest_dir)
+            and mode & self.CopyMode.symlink
+        ):
             supported_modes -= self.CopyMode.symlink
             constraint = (
                 f"Destination directory is on CIFS mount ({dest_dir}) "

fileformats/core/fs_mount_identifier.py

@@ -1,6 +1,8 @@
 import os
 import typing as ty
+import string
 from pathlib import Path
+import platform
 import re
 from contextlib import contextmanager
 import subprocess as sp
@@ -12,7 +14,7 @@ class FsMountIdentifier:
     features that can be used (e.g. symlinks)"""
 
     @classmethod
-    def on_cifs(cls, path: os.PathLike) -> bool:
+    def symlinks_supported(cls, path: os.PathLike) -> bool:
         """
         Check whether a file path is on a CIFS filesystem mounted in a POSIX host.
 
@@ -27,12 +29,12 @@ def on_cifs(cls, path: os.PathLike) -> bool:
 
         This check is written to support disabling symlinks on CIFS shares.
 
-        NB: This function and sub-functions are copied from the nipype.utils.filemanip module
+        NB: This function and sub-functions are modified from the nipype.utils.filemanip module
 
 
         NB: Adapted from https://github.com/nipy/nipype
         """
-        return cls.get_mount(path)[1] == "cifs"
+        return cls.get_mount(path)[1] != "cifs"
 
     @classmethod
     def on_same_mount(cls, path1: os.PathLike, path2: os.PathLike) -> bool:
@@ -54,19 +56,21 @@ def get_mount(cls, path: os.PathLike) -> ty.Tuple[Path, str]:
             the root of the mount the path sits on
         fstype : str
             the type of the file-system (e.g. ext4 or cifs)"""
-        try:
-            # Only the first match (most recent parent) counts, mount table sorted longest
-            # to shortest
-            return next(
-                (Path(p), t)
-                for p, t in cls.get_mount_table()
-                if str(path).startswith(p)
+        strpath = str(Path(path).absolute())
+        mount_table = cls.get_mount_table()
+        matches = sorted(
+            ((Path(p), t) for p, t in mount_table if strpath.startswith(p)),
+            key=lambda m: len(str(m[0])),
+        )
+        if not matches:
+            raise ValueError(
+                f"Path {strpath} is not on a known mount point:\n{mount_table}"
             )
-        except StopIteration:
-            return (Path("/"), "ext4")
+        # return mount point with longest matching prefix
+        return matches[-1]
 
     @classmethod
-    def generate_cifs_table(cls) -> ty.List[ty.Tuple[str, str]]:
+    def generate_mount_table(cls) -> ty.List[ty.Tuple[str, str]]:
         """
         Construct a reverse-length-ordered list of mount points that fall under a CIFS mount.
 
@@ -76,7 +80,25 @@ def generate_cifs_table(cls) -> ty.List[ty.Tuple[str, str]]:
         empty list.
 
         """
+        if platform.system() == "Windows":
+            drive_names = [
+                c + ":" for c in string.ascii_uppercase if os.path.exists(c + ":")
+            ]
+            drives = []
+            for drive_name in drive_names:
+                result = sp.run(
+                    ["fsutil", "fsinfo", "fstype", drive_name],
+                    capture_output=True,
+                    text=True,
+                )
+                fstype = result.stdout.strip().split(" ")[-1].lower()
+                drives.append((drive_name, fstype))
+            return drives
         exit_code, output = sp.getstatusoutput("mount")
+        if exit_code != 0:
+            raise RuntimeError(
+                "Failed to get mount table (exit code {}): {}".format(exit_code, output)
+            )
         return cls.parse_mount_table(exit_code, output)
 
     @classmethod
@@ -90,10 +112,6 @@ def parse_mount_table(
         outputs
 
         """
-        # Not POSIX
-        if exit_code != 0:
-            return []
-
         # Linux mount example:  sysfs on /sys type sysfs (rw,nosuid,nodev,noexec)
         #                          <PATH>^^^^      ^^^^^<FSTYPE>
         # OSX mount example:    /dev/disk2 on / (hfs, local, journaled)
@@ -105,28 +123,23 @@ def parse_mount_table(
         matches = [(ll, pattern.match(ll)) for ll in output.strip().splitlines() if ll]
 
         # (path, fstype) tuples, sorted by path length (longest first)
-        mount_info = sorted(
+        mounts = sorted(
             (match.groups() for _, match in matches if match is not None),
             key=lambda x: len(x[0]),
             reverse=True,
         )
-        cifs_paths = [path for path, fstype in mount_info if fstype.lower() == "cifs"]
 
         # Report failures as warnings
         for line, match in matches:
             if match is None:
                 logger.debug("Cannot parse mount line: '%s'", line)
 
-        return [
-            mount
-            for mount in mount_info
-            if any(mount[0].startswith(path) for path in cifs_paths)
-        ]
+        return mounts
 
     @classmethod
     def get_mount_table(cls) -> ty.List[ty.Tuple[str, str]]:
         if cls._mount_table is None:
-            cls._mount_table = cls.generate_cifs_table()
+            cls._mount_table = cls.generate_mount_table()
         return cls._mount_table
 
     @classmethod

fileformats/core/tests/test_fs_mount_identifier.py

@@ -1,4 +1,5 @@
 import os.path
+import platform
 import pytest
 from fileformats.core.fs_mount_identifier import FsMountIdentifier
 from fileformats.generic import File
@@ -33,7 +34,32 @@
 gvfsd-fuse on /run/user/1002/gvfs type fuse.gvfsd-fuse (rw,nosuid,nodev,relatime,user_id=1002,group_id=1002)
 """,
         0,
-        [],
+        [
+            ("/sys/fs/cgroup/cpu,cpuacct", "cgroup"),
+            ("/sys/firmware/efi/efivars", "efivarfs"),
+            ("/proc/sys/fs/binfmt_misc", "autofs"),
+            ("/sys/fs/fuse/connections", "fusectl"),
+            ("/sys/fs/cgroup/systemd", "cgroup"),
+            ("/sys/fs/cgroup/freezer", "cgroup"),
+            ("/sys/fs/cgroup/cpuset", "cgroup"),
+            ("/sys/kernel/security", "securityfs"),
+            ("/var/lib/docker/aufs", "ext4"),
+            ("/sys/fs/cgroup/pids", "cgroup"),
+            ("/run/user/1002/gvfs", "fuse.gvfsd-fuse"),
+            ("/sys/kernel/debug", "debugfs"),
+            ("/sys/fs/cgroup", "tmpfs"),
+            ("/sys/fs/pstore", "pstore"),
+            ("/dev/hugepages", "hugetlbfs"),
+            ("/dev/mqueue", "mqueue"),
+            ("/boot/efi", "vfat"),
+            ("/dev/pts", "devpts"),
+            ("/dev/shm", "tmpfs"),
+            ("/proc", "proc"),
+            ("/sys", "sysfs"),
+            ("/dev", "devtmpfs"),
+            ("/run", "tmpfs"),
+            ("/", "ext4"),
+        ],
     ),
     # OS X, no CIFS
     (
@@ -53,7 +79,22 @@
 /dev/disk1s3 on /Volumes/Boot OS X (hfs, local, journaled, nobrowse)
 """,
         0,
-        [],
+        [
+            ("/Volumes/MyBookData", "hfs"),
+            ("/Volumes/AFNI_SHARE", "nfs"),
+            ("/Volumes/Boot OS X", "hfs"),
+            ("/Volumes/INCOMING", "nfs"),
+            ("/Volumes/raid.bot", "nfs"),
+            ("/Volumes/raid.top", "autofs"),
+            ("/Network/Servers", "autofs"),
+            ("/Volumes/safni", "autofs"),
+            ("/Volumes/afni", "nfs"),
+            ("/Volumes/afni", "nfs"),
+            ("/home", "autofs"),
+            ("/dev", "devfs"),
+            ("/net", "autofs"),
+            ("/", "hfs"),
+        ],
     ),
     # Non-zero exit code
     ("", 1, []),
@@ -85,7 +126,32 @@
 gvfsd-fuse on /run/user/1002/gvfs type fuse.gvfsd-fuse (rw,nosuid,nodev,relatime,user_id=1002,group_id=1002)
 """,
         0,
-        [],
+        [
+            ("/sys/fs/cgroup/cpu,cpuacct", "cgroup"),
+            ("/sys/firmware/efi/efivars", "efivarfs"),
+            ("/proc/sys/fs/binfmt_misc", "autofs"),
+            ("/sys/fs/fuse/connections", "fusectl"),
+            ("/sys/fs/cgroup/systemd", "cgroup"),
+            ("/sys/fs/cgroup/freezer", "cgroup"),
+            ("/sys/fs/cgroup/cpuset", "cgroup"),
+            ("/sys/kernel/security", "securityfs"),
+            ("/var/lib/docker/aufs", "ext4"),
+            ("/sys/fs/cgroup/pids", "cgroup"),
+            ("/run/user/1002/gvfs", "fuse.gvfsd-fuse"),
+            ("/sys/kernel/debug", "debugfs"),
+            ("/sys/fs/cgroup", "tmpfs"),
+            ("/sys/fs/pstore", "pstore"),
+            ("/dev/hugepages", "hugetlbfs"),
+            ("/dev/mqueue", "mqueue"),
+            ("/boot/efi", "vfat"),
+            ("/dev/pts", "devpts"),
+            ("/dev/shm", "tmpfs"),
+            ("/proc", "proc"),
+            ("/sys", "sysfs"),
+            ("/dev", "devtmpfs"),
+            ("/run", "tmpfs"),
+            ("/", "ext4"),
+        ],
     ),
     # Variant of OS X example with CIFS added manually
     (
@@ -98,7 +164,15 @@
 elros:/volume2/AFNI_SHARE on /Volumes/AFNI_SHARE (nfs)
 """,
         0,
-        [("/Volumes/afni/fraid", "nfs"), ("/Volumes/afni", "cifs")],
+        [
+            ("/Volumes/afni/fraid", "nfs"),
+            ("/Volumes/AFNI_SHARE", "nfs"),
+            ("/Volumes/INCOMING", "nfs"),
+            ("/Volumes/raid.bot", "nfs"),
+            ("/Volumes/afni", "cifs"),
+            ("/dev", "devfs"),
+            ("/", "hfs"),
+        ],
     ),
     # From Windows: docker run --rm -it -v C:\:/data busybox mount
     (
@@ -140,7 +214,44 @@
 tmpfs on /sys/firmware type tmpfs (ro,relatime)
 """,
         0,
-        [("/data", "cifs")],
+        [
+            ("/sys/fs/cgroup/perf_event", "cgroup"),
+            ("/sys/fs/cgroup/net_prio", "cgroup"),
+            ("/sys/fs/cgroup/cpuacct", "cgroup"),
+            ("/sys/fs/cgroup/devices", "cgroup"),
+            ("/sys/fs/cgroup/freezer", "cgroup"),
+            ("/sys/fs/cgroup/net_cls", "cgroup"),
+            ("/sys/fs/cgroup/hugetlb", "cgroup"),
+            ("/sys/fs/cgroup/systemd", "cgroup"),
+            ("/sys/fs/cgroup/cpuset", "cgroup"),
+            ("/sys/fs/cgroup/memory", "cgroup"),
+            ("/sys/fs/cgroup/blkio", "cgroup"),
+            ("/sys/fs/cgroup/pids", "cgroup"),
+            ("/proc/sysrq-trigger", "proc"),
+            ("/sys/fs/cgroup/cpu", "cgroup"),
+            ("/proc/sched_debug", "tmpfs"),
+            ("/etc/resolv.conf", "ext4"),
+            ("/proc/timer_list", "tmpfs"),
+            ("/sys/fs/cgroup", "tmpfs"),
+            ("/etc/hostname", "ext4"),
+            ("/sys/firmware", "tmpfs"),
+            ("/dev/console", "devpts"),
+            ("/dev/mqueue", "mqueue"),
+            ("/proc/kcore", "tmpfs"),
+            ("/etc/hosts", "ext4"),
+            ("/proc/scsi", "tmpfs"),
+            ("/proc/bus", "proc"),
+            ("/proc/irq", "proc"),
+            ("/proc/sys", "proc"),
+            ("/dev/pts", "devpts"),
+            ("/dev/shm", "tmpfs"),
+            ("/proc/fs", "proc"),
+            ("/proc", "proc"),
+            ("/data", "cifs"),
+            ("/dev", "tmpfs"),
+            ("/sys", "sysfs"),
+            ("/", "overlay"),
+        ],
     ),
     # From @yarikoptic - added blank lines to test for resilience
     (
@@ -153,7 +264,13 @@
 
 """,
         0,
-        [],
+        [
+            ("/dev/ptmx", "devpts"),
+            ("/dev/shm", "tmpfs"),
+            ("/dev/pts", "devpts"),
+            ("/proc", "proc"),
+            ("/sys", "sysfs"),
+        ],
     ),
 )
 
@@ -163,27 +280,24 @@ def test_parse_mount_table(output, exit_code, expected):
     assert FsMountIdentifier.parse_mount_table(exit_code, output) == expected
 
 
-def test_cifs_check():
-    assert isinstance(FsMountIdentifier.get_mount_table(), list)
-    assert isinstance(FsMountIdentifier.on_cifs("/"), bool)
-    fake_table = [("/scratch/tmp", "ext4"), ("/scratch", "cifs")]
+@pytest.mark.skipif(
+    platform.system() == "Windows", reason="Windows does not have mount table"
+)
+def test_mount_check():
+    fake_table = [("/", "ext4"), ("/scratch/tmp", "ext4"), ("/scratch", "cifs")]
     cifs_targets = [
-        ("/scratch/tmp/x/y", False),
-        ("/scratch/tmp/x", False),
-        ("/scratch/x/y", True),
-        ("/scratch/x", True),
-        ("/x/y", False),
-        ("/x", False),
-        ("/", False),
+        ("/scratch/tmp/x/y", True),
+        ("/scratch/tmp/x", True),
+        ("/scratch/x/y", False),
+        ("/scratch/x", False),
+        ("/x/y", True),
+        ("/x", True),
+        ("/", True),
     ]
 
-    with FsMountIdentifier.patch_table([]):
-        for target, _ in cifs_targets:
-            assert FsMountIdentifier.on_cifs(target) is False
-
     with FsMountIdentifier.patch_table(fake_table):
         for target, expected in cifs_targets:
-            assert FsMountIdentifier.on_cifs(target) is expected
+            assert FsMountIdentifier.symlinks_supported(target) is expected
 
 
 def test_copy_constraints(tmp_path):
@@ -242,3 +356,18 @@ def test_copy_constraints(tmp_path):
         assert (
             os.stat(ext4_file).st_ino != os.stat(ext4_file_on_cifs).st_ino
         )  # Not hardlink
+
+
+def test_generate_mount_table():
+    mount_table = FsMountIdentifier.get_mount_table()
+    assert isinstance(mount_table, list)
+    # We can't test the actual mount table, but we can test that the function actually
+    # runs and returns at least one mount/drive
+    assert mount_table
+
+
+@pytest.mark.skipif(
+    platform.system() == "Windows", reason="Windows does not have mount table"
+)
+def test_symlink_supported():
+    assert isinstance(FsMountIdentifier.symlinks_supported("/"), bool)