Merge pull request #118 from ArcanaFramework/updates-convertible-from

added write_text method to UnicodeFile

Author: tclose

docs/source/developer/extras.rst

@@ -134,7 +134,6 @@ to do a generic conversion between all image types,
     from pathlib import Path
     import tempfile
     import pydra.mark
-    import pydra.engine.specs
     from fileformats.core import converter
     from .raster import RasterImage, Bitmap, Gif, Jpeg, Png, Tiff
 
@@ -144,9 +143,8 @@ to do a generic conversion between all image types,
     @converter(target_format=Jpeg, output_format=Jpeg)
     @converter(target_format=Png, output_format=Png)
     @converter(target_format=Tiff, output_format=Tiff)
-    @pydra.mark.task
-    @pydra.mark.annotate({"return": {"out_file": RasterImage}})
-    def convert_image(in_file: RasterImage, output_format: type, out_dir: ty.Optional[Path] = None):
+    @python.define(outputs=["out_file"])
+    def convert_image(in_file: RasterImage, output_format: type, out_dir: Path | None = None) -> RasterImage:
         data_array = in_file.load()
         if out_dir is None:
             out_dir = Path(tempfile.mkdtemp())

extras/fileformats/extras/core/tests/test_converter.py

@@ -1,14 +1,22 @@
-import typing as ty
-import attrs
 from pathlib import Path
+
+import attrs
 import pytest
 from pydra.compose import python, shell
-from fileformats.generic import File, FileSet
-from fileformats.testing import Foo, Bar, Baz, Qux
-from fileformats.testing import ConvertibleToFile, ConcreteClass, AnotherConcreteClass
+
+from conftest import write_test_file
 from fileformats.core import converter
 from fileformats.core.exceptions import FormatConversionError
-from conftest import write_test_file
+from fileformats.generic import File, FileSet
+from fileformats.testing import (
+    AnotherConcreteClass,
+    Bar,
+    Baz,
+    ConcreteClass,
+    ConvertibleToFile,
+    Foo,
+    Qux,
+)
 
 
 @converter
@@ -95,13 +103,13 @@ def test_convert_mapped_conversion(work_dir):
 @pytest.mark.parametrize(
     ["klass", "convertible_from"],
     [
-        [Bar, ty.Union[Bar, Baz, Foo]],
-        [Qux, ty.Union[Foo, Qux]],
+        [Bar, Bar | Baz | Foo],
+        [Qux, Foo | Qux],
         [Foo, Foo],
         [Baz, Baz],
         [
             ConvertibleToFile,
-            ty.Union[AnotherConcreteClass, ConcreteClass, ConvertibleToFile],
+            AnotherConcreteClass | ConcreteClass | ConvertibleToFile,
         ],
     ],
 )

extras/fileformats/extras/testing/__init__.py

@@ -1,9 +1,33 @@
-from fileformats.core import converter
-from fileformats.testing import AbstractFile, ConvertibleToFile
 from pydra.compose import python
 
+from fileformats.core import converter
+from fileformats.testing import AbstractFile, ConvertibleToFile, EncodedText
+from fileformats.text import TextFile
+
 
 @converter  # pyright: ignore[reportArgumentType]
 @python.define(outputs=["out_file"])  # type: ignore[misc]
 def ConvertibleToConverter(in_file: AbstractFile) -> ConvertibleToFile:
     return ConvertibleToFile.sample()
+
+
+@converter  # pyright: ignore[reportArgumentType]
+@python.define(outputs=["out_file"])  # type: ignore[misc]
+def EncodedFromTextConverter(in_file: TextFile) -> EncodedText:
+    contents = in_file.read_text()
+    # Encode by shifting ASCII codes forward by 1
+    encoded_contents = "".join(chr(ord(c) + 1) for c in contents)
+    out_file = EncodedText.sample()
+    out_file.write_text(encoded_contents)
+    return out_file
+
+
+@converter  # pyright: ignore[reportArgumentType]
+@python.define(outputs=["out_file"])  # type: ignore[misc]
+def EncodedToTextConverter(in_file: EncodedText) -> TextFile:
+    contents = in_file.read_text()
+    # Decode by shifting ASCII codes back by 1
+    decoded_contents = "".join(chr(ord(c) - 1) for c in contents)
+    out_file = TextFile.sample()
+    out_file.write_text(decoded_contents)
+    return out_file

extras/fileformats/extras/text/__init__.py

@@ -1 +1,3 @@
-from . import load_save  # noqa: F401
+from . import load_save
+
+__all__ = ["load_save"]

extras/pyproject.toml

@@ -44,9 +44,13 @@ application = [
     "pydicom >=2.3",
     "medimages4tests",
 ]
+audio = []
 image = [
     "imageio >=2.24.0",
 ]
+model = []
+text = []
+video = []
 vendor_openxmlformats_officedocument = [
     "python-docx >= 1.2.0",
 ]

fileformats/core/__init__.py

@@ -1,8 +1,8 @@
 from ._version import __version__
 from .classifier import Classifier
-from .datatype import DataType
+from .datatype import DataType, FieldPrimitive
 from .mock import MockMixin
-from .fileset import FileSet
+from .fileset import FileSet, FileSetPrimitive
 from .field import Field
 from .identification import (
     to_mime,
@@ -19,6 +19,8 @@
     "Classifier",
     "DataType",
     "FileSet",
+    "FieldPrimitive",
+    "FileSetPrimitive",
     "MockMixin",
     "Field",
     "to_mime",

fileformats/core/datatype.py

@@ -1,11 +1,18 @@
 from __future__ import annotations
 
+import decimal
 import importlib
 import itertools
+import sys
 import typing as ty
 from abc import ABCMeta
 from inspect import isclass
 
+if sys.version_info >= (3, 10):
+    from typing import TypeAlias
+else:
+    from typing_extensions import TypeAlias
+
 from fileformats.core.typing import Self
 
 from .classifier import Classifier
@@ -26,6 +33,19 @@
 if ty.TYPE_CHECKING:
     from .converter_helpers import Converter
 
+FieldPrimitive: TypeAlias = ty.Union[
+    str,
+    int,
+    float,
+    bool,
+    decimal.Decimal,
+    ty.Sequence[str],
+    ty.Sequence[int],
+    ty.Sequence[float],
+    ty.Sequence[bool],
+    ty.Sequence[decimal.Decimal],
+]
+
 
 class DataType(Classifier, metaclass=ABCMeta):
     """
@@ -78,9 +98,9 @@ def all_types(self) -> ty.Iterator[ty.Type[DataType]]:
         return itertools.chain(FileSet.all_formats, Field.all_fields)
 
     @classmethod
-    def subclasses(cls) -> ty.Generator[ty.Type[Self], None, None]:
+    def subclasses(cls, **kwargs: ty.Any) -> ty.Generator[ty.Type[Self], None, None]:
         """Iterate over all installed subclasses"""
-        for subpkg in subpackages():
+        for subpkg in subpackages(**kwargs):
             for attr_name in dir(subpkg):
                 attr = getattr(subpkg, attr_name)
                 if (
@@ -95,7 +115,7 @@ def subclasses(cls) -> ty.Generator[ty.Type[Self], None, None]:
     def get_converter(
         cls,
         source_format: ty.Type[DataType],
-    ) -> "Converter | None":
+    ) -> "ty.Optional[Converter]":
         if issubclass(source_format, cls):
             return None
         else:

fileformats/core/fileset.py

@@ -4,9 +4,11 @@
 import inspect
 import itertools
 import logging
+import operator
 import os
 import shutil
 import struct
+import sys
 import tempfile
 import typing as ty
 from collections import Counter
@@ -16,7 +18,13 @@
 from pathlib import Path
 from warnings import warn
 
+if sys.version_info >= (3, 10):
+    from typing import TypeAlias
+else:
+    from typing_extensions import TypeAlias
+
 from fileformats.core.typing import Self
+from fileformats.core.utils import _excluded_subpackages
 
 from .classifier import Classifier
 from .datatype import DataType
@@ -43,6 +51,19 @@
     from .converter_helpers import Converter
 
 
+class SupportsDunderLT(ty.Protocol):
+    def __lt__(self, other: ty.Any) -> bool:
+        ...
+
+
+class SupportsDunderGT(ty.Protocol):
+    def __gt__(self, other: ty.Any) -> bool:
+        ...
+
+
+FileSetPrimitive: TypeAlias = ty.Union[os.PathLike[str], ty.Sequence[os.PathLike[str]]]
+
+
 FILE_CHUNK_LEN_DEFAULT = 8192
 
 
@@ -542,7 +563,7 @@ def convert(
     def get_converter(
         cls,
         source_format: ty.Type[DataType],
-    ) -> "Converter | None":
+    ) -> "ty.Optional[Converter]":
         """Get a converter that converts from the source format type
         into the format specified by the class
 
@@ -572,12 +593,9 @@ def get_converter(
             return None
         # trigger loading of standard converters for target format
         converters = cls.get_converters_dict()
-        try:
-            unclassified = source_format.unclassified  # type: ignore
-        except AttributeError:
-            import_extras_module(source_format)
-        else:
-            import_extras_module(unclassified)
+        # import extras modules
+        source_format._import_extras_module()  # type: ignore[attr-defined]
+        cls._import_extras_module()
         try:
             converter = converters[source_format]
         except KeyError:
@@ -613,6 +631,15 @@ def get_converter(
             converters[source_format] = converter
         return converter
 
+    @classmethod
+    def _import_extras_module(cls) -> None:
+        try:
+            unclassified = cls.unclassified  # type: ignore
+        except AttributeError:
+            import_extras_module(cls)
+        else:
+            import_extras_module(unclassified)
+
     @classmethod
     def get_converters_dict(
         cls, klass: ty.Optional[ty.Type[DataType]] = None
@@ -633,15 +660,17 @@ def get_converters_dict(
     @classmethod
     def convertible_from(
         cls,
-        only_namespace_parents: bool = True,
-        union_sort_key: ty.Callable[[DataType], ty.Any] = attrgetter("__name__"),
+        union_sort_key: ty.Callable[
+            [ty.Type[DataType]],
+            ty.Union[SupportsDunderLT, SupportsDunderGT],
+        ] = attrgetter("__name__"),
     ) -> ty.Type["DataType"]:
         """Union of types that can be converted to this type, including the current type.
         If there are no other types that can be converted to this type, return the current type
 
         Parameters
         ----------
-        only_namespace_parents: bool
+        include_generic: bool
             If True, only consider parent classes in the same namespace for conversion.
         union_sort_key : callable[[DataType], Any], optional
             A function used to sort the union of types. Defaults to sorting by the type name.
@@ -652,13 +681,14 @@ def convertible_from(
             The type or union of types that can be converted to this type.
         """
 
-        ns = cls.namespace
         datatypes: ty.List[ty.Type[DataType]] = [cls]
-        for fformat in FileSet.subclasses():
-            if issubclass(cls, fformat) and (
-                fformat.namespace == ns or not only_namespace_parents
-            ):
-                datatypes.extend(fformat.get_converters_dict().keys())
+        cls._import_extras_module()
+        exclude_subpackages = copy(_excluded_subpackages)
+        exclude_subpackages.discard(cls.namespace)
+        for subcls in FileSet.subclasses(exclude=exclude_subpackages):
+            if issubclass(subcls, cls):
+                subcls._import_extras_module()
+                datatypes.extend(subcls.get_converters_dict().keys())
         if len(datatypes) == 1:
             return cls
         concrete_datatypes = set()
@@ -675,8 +705,8 @@ def add_concrete(datatype: ty.Type[DataType]) -> None:
         for datatype in datatypes:
             add_concrete(datatype)
         # TODO: Might want to sort datatypes in a more specific order
-        return ty.Union.__getitem__(  # pyright: ignore[reportAttributeAccessIssue]
-            tuple(sorted(concrete_datatypes, key=union_sort_key))
+        return functools.reduce(
+            operator.or_, sorted(concrete_datatypes, key=union_sort_key)
         )  # type: ignore[return-value]
 
     @classmethod

fileformats/core/identification.py

@@ -1,3 +1,4 @@
+import functools
 import inspect
 import operator
 import re
@@ -102,7 +103,7 @@ def from_mime(
             item_mime = item_mime[1:-1]
         return ty.List[from_mime(item_mime)]  # type: ignore
     if "," in mime_str:
-        return ty.Union.__getitem__(tuple(from_mime(t) for t in mime_str.split(",")))  # type: ignore
+        return functools.reduce(operator.or_, (from_mime(t) for t in mime_str.split(",")))  # type: ignore
     return fileformats.core.DataType.from_mime(mime_str)