diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a23b7fdcd..a1f6be269 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,7 +20,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.13.2 hooks: - - id: ruff + - id: ruff-check # - repo: https://github.com/econchick/interrogate # rev: 1.5.0 # hooks: diff --git a/pyproject.toml b/pyproject.toml index 034fb5643..2c4e22de4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ dependencies = [ 'numpy>=1.19.3', "pandas>=1.2.0", "ruamel.yaml>=0.16", + "pydantic>=2.0.0", ] dynamic = ["version"] @@ -84,6 +85,7 @@ min-reqs = [ "numpy==1.19.3", "pandas==1.2.0", "ruamel.yaml==0.16.0", + "pydantic==2.0.0", "scipy==1.7.0", "tqdm==4.41.0", "zarr==2.12.0", @@ -151,11 +153,11 @@ omit = [ "*/hdmf/testing/*", ] -# [tool.black] -# line-length = 120 -# preview = true -# exclude = ".git|.mypy_cache|.tox|.venv|venv|.ipynb_checkpoints|_build/|dist/|__pypackages__|.ipynb" -# force-exclude = "src/hdmf/common/hdmf-common-schema|docs/gallery" +[tool.black] +line-length = 120 +preview = true +exclude = ".git|.mypy_cache|.tox|.venv|venv|.ipynb_checkpoints|_build/|dist/|__pypackages__|.ipynb" +force-exclude = "src/hdmf/common/hdmf-common-schema|docs/gallery" [tool.ruff] lint.select = ["E", "F", "T100", "T201", "T203", "C901"] diff --git a/src/hdmf/build/classgenerator.py b/src/hdmf/build/classgenerator.py index dcdb49b77..853b720b3 100644 --- a/src/hdmf/build/classgenerator.py +++ b/src/hdmf/build/classgenerator.py @@ -5,8 +5,7 @@ import numpy as np from ..container import Container, Data, MultiContainerInterface -from ..spec import AttributeSpec, LinkSpec, RefSpec, GroupSpec -from ..spec.spec import BaseStorageSpec, ZERO_OR_MANY, ONE_OR_MANY +from ..spec import AttributeSpec, LinkSpec, RefSpec, GroupSpec, BaseStorageSpec from ..utils import docval, getargs, ExtenderMeta, get_docval, popargs, AllowPositional @@ -379,7 +378,7 @@ class MCIClassGenerator(CustomClassGenerator): @classmethod def apply_generator_to_field(cls, field_spec, bases, type_map): """Return True if the field spec has quantity * or +, False otherwise.""" - return getattr(field_spec, 'quantity', None) in (ZERO_OR_MANY, ONE_OR_MANY) + return isinstance(field_spec, (BaseStorageSpec, LinkSpec)) and field_spec.is_many() @classmethod def process_field_spec(cls, classdict, docval_args, parent_cls, attr_name, not_inherited_fields, type_map, spec): diff --git a/src/hdmf/build/manager.py b/src/hdmf/build/manager.py index aa66c0862..4e90e0639 100644 --- a/src/hdmf/build/manager.py +++ b/src/hdmf/build/manager.py @@ -7,8 +7,7 @@ from .classgenerator import ClassGeneratorManager, CustomClassGenerator, MCIClassGenerator from ..container import AbstractContainer, Container, Data from ..term_set import TypeConfigurator -from ..spec import DatasetSpec, GroupSpec, NamespaceCatalog, RefSpec -from ..spec.spec import BaseStorageSpec +from ..spec import BaseStorageSpec, DatasetSpec, GroupSpec, NamespaceCatalog, RefSpec from ..utils import docval, getargs, ExtenderMeta, get_docval diff --git a/src/hdmf/build/objectmapper.py b/src/hdmf/build/objectmapper.py index e01eabf11..0dba10f69 100644 --- a/src/hdmf/build/objectmapper.py +++ b/src/hdmf/build/objectmapper.py @@ -19,8 +19,7 @@ from ..term_set import TermSetWrapper from ..data_utils import DataIO, AbstractDataChunkIterator from ..query import ReferenceResolver -from ..spec import Spec, AttributeSpec, DatasetSpec, GroupSpec, LinkSpec, RefSpec -from ..spec.spec import BaseStorageSpec +from ..spec import BaseStorageSpec, Spec, AttributeSpec, DatasetSpec, GroupSpec, LinkSpec, RefSpec from ..utils import docval, getargs, ExtenderMeta, get_docval, get_data_shape, StrDataset _const_arg = '__constructor_arg' diff --git a/src/hdmf/spec/__init__.py b/src/hdmf/spec/__init__.py index 09ad6d073..390f065de 100644 --- a/src/hdmf/spec/__init__.py +++ b/src/hdmf/spec/__init__.py @@ -1,5 +1,9 @@ from .catalog import SpecCatalog from .namespace import NamespaceCatalog, SpecNamespace, SpecReader -from .spec import (AttributeSpec, DatasetSpec, DtypeHelper, DtypeSpec, GroupSpec, LinkSpec, - NAME_WILDCARD, RefSpec, Spec) +# from .spec import (AttributeSpec, DatasetSpec, DtypeHelper, DtypeSpec, GroupSpec, LinkSpec, +# NAME_WILDCARD, RefSpec, Spec) +from .spec2 import ( + AttributeSpec, DatasetSpec, DtypeHelper, DtypeSpec, GroupSpec, LinkSpec, RefSpec, Spec, BaseStorageSpec, + QuantityEnum +) from .write import NamespaceBuilder, SpecWriter, export_spec diff --git a/src/hdmf/spec/catalog.py b/src/hdmf/spec/catalog.py index c40d2130c..8c42d829d 100644 --- a/src/hdmf/spec/catalog.py +++ b/src/hdmf/spec/catalog.py @@ -2,7 +2,7 @@ import warnings from collections import OrderedDict -from .spec import BaseStorageSpec, GroupSpec +from .spec2 import BaseStorageSpec, GroupSpec from ..utils import docval, getargs diff --git a/src/hdmf/spec/namespace.py b/src/hdmf/spec/namespace.py index dc7d238d1..3cc2c7156 100644 --- a/src/hdmf/spec/namespace.py +++ b/src/hdmf/spec/namespace.py @@ -9,7 +9,7 @@ import graphlib from .catalog import SpecCatalog -from .spec import DatasetSpec, GroupSpec, BaseStorageSpec +from .spec2 import DatasetSpec, GroupSpec, BaseStorageSpec from ..utils import docval, getargs, popargs, get_docval, is_newer_version _namespace_args = [ @@ -567,9 +567,9 @@ def __register_type(self, ndt, inc_ns, catalog, registered_types): spec_file = inc_ns.catalog.get_spec_source_file(ndt) self.__register_dependent_types(spec, inc_ns, catalog, registered_types) if isinstance(spec, DatasetSpec): - built_spec = self.dataset_spec_cls.build_spec(spec) + built_spec = spec # TODO self.dataset_spec_cls.build_spec(spec) else: - built_spec = self.group_spec_cls.build_spec(spec) + built_spec = spec # TODO self.group_spec_cls.build_spec(spec) registered_types.add(ndt) catalog.register_spec(built_spec, spec_file) diff --git a/src/hdmf/spec/oldspec.py b/src/hdmf/spec/oldspec.py new file mode 100644 index 000000000..6bb761663 --- /dev/null +++ b/src/hdmf/spec/oldspec.py @@ -0,0 +1,1578 @@ +from abc import ABCMeta +from collections import OrderedDict +from copy import copy +from itertools import chain +from typing import Union, TYPE_CHECKING +from warnings import warn + +from ..utils import docval, getargs, popargs, get_docval + +if TYPE_CHECKING: + from .namespace import SpecNamespace # noqa: F401 + +NAME_WILDCARD = None # this is no longer used, but kept for backward compatibility +ZERO_OR_ONE = '?' +ZERO_OR_MANY = '*' +ONE_OR_MANY = '+' +DEF_QUANTITY = 1 +FLAGS = { + 'zero_or_one': ZERO_OR_ONE, + 'zero_or_many': ZERO_OR_MANY, + 'one_or_many': ONE_OR_MANY +} + + +class DtypeHelper: + # Dict where the keys are the primary data type and the values are list of strings with synonyms for the dtype + # make sure keys are consistent between hdmf.spec.spec.DtypeHelper.primary_dtype_synonyms, + # hdmf.build.objectmapper.ObjectMapper.__dtypes, hdmf.build.manager.TypeMap._spec_dtype_map, + # hdmf.validate.validator.__allowable, and backend dtype maps + # see https://hdmf-schema-language.readthedocs.io/en/latest/description.html#dtype + primary_dtype_synonyms = { + 'float': ["float", "float32"], + 'double': ["double", "float64"], + 'short': ["int16", "short"], + 'int': ["int32", "int"], + 'long': ["int64", "long"], + 'utf': ["text", "utf", "utf8", "utf-8"], + 'ascii': ["ascii", "bytes"], + 'bool': ["bool"], + 'int8': ["int8"], + 'uint8': ["uint8"], + 'uint16': ["uint16"], + 'uint32': ["uint32", "uint"], + 'uint64': ["uint64"], + 'object': ['object'], + 'numeric': ['numeric'], + 'isodatetime': ["isodatetime", "datetime", "date"] + } + + # List of recommended primary dtype strings. These are the keys of primary_dtype_string_synonyms + recommended_primary_dtypes = list(primary_dtype_synonyms.keys()) + + # List of valid primary data type strings + valid_primary_dtypes = set(list(primary_dtype_synonyms.keys()) + + [vi for v in primary_dtype_synonyms.values() for vi in v]) + + @staticmethod + def simplify_cpd_type(cpd_type): + ''' + Transform a list of DtypeSpecs into a list of strings. + Use for simple representation of compound type and + validation. + + :param cpd_type: The list of DtypeSpecs to simplify + :type cpd_type: list + + ''' + ret = list() + for exp in cpd_type: + exp_key = exp.dtype + if isinstance(exp_key, RefSpec): + exp_key = exp_key.reftype + ret.append(exp_key) + return ret + + @staticmethod + def check_dtype(dtype): + """Check that the dtype string is a reference or a valid primary dtype.""" + if not isinstance(dtype, RefSpec) and dtype not in DtypeHelper.valid_primary_dtypes: + raise ValueError("dtype '%s' is not a valid primary data type. Allowed dtypes: %s" + % (dtype, str(DtypeHelper.valid_primary_dtypes))) + return dtype + + # all keys and values should be keys in primary_dtype_synonyms + additional_allowed = { + 'float': ['double'], + 'int8': ['short', 'int', 'long'], + 'short': ['int', 'long'], + 'int': ['long'], + 'uint8': ['uint16', 'uint32', 'uint64'], + 'uint16': ['uint32', 'uint64'], + 'uint32': ['uint64'], + 'utf': ['ascii'] + } + + # if the spec dtype is a key in __allowable, then all types in __allowable[key] are valid + allowable = dict() + for dt, dt_syn in primary_dtype_synonyms.items(): + allow = copy(dt_syn) + if dt in additional_allowed: + for addl in additional_allowed[dt]: + allow.extend(primary_dtype_synonyms[addl]) + for syn in dt_syn: + allowable[syn] = allow + allowable['numeric'].extend(set(chain.from_iterable(v for k, v in allowable.items() if 'int' in k or 'float' in k))) + + @staticmethod + def is_allowed_dtype(new: str, orig: str): + if orig not in DtypeHelper.allowable: + raise ValueError(f"Unknown dtype '{orig}'") + return new in DtypeHelper.allowable[orig] + + +def _is_sub_dtype(new: Union[str, "RefSpec"], orig: Union[str, "RefSpec"]): + if isinstance(orig, RefSpec) != isinstance(new, RefSpec): + return False + + if isinstance(orig, RefSpec): # both are RefSpec + # check ref target is a subtype of the original ref target + # TODO: implement subtype check for RefSpec. might need to resolve RefSpec target type to a spec first + # return orig == new + return True + else: + return DtypeHelper.is_allowed_dtype(new, orig) + + +def _resolve_inc_spec_dtype( + spec: Union['AttributeSpec', 'DatasetSpec'], + inc_spec: Union['AttributeSpec', 'DatasetSpec'] + ): + if inc_spec.dtype is None: + # nothing to include/check + return + + if spec.dtype is None: + # no dtype defined, just use the included spec dtype + spec['dtype'] = inc_spec.dtype + return + + # both inc_spec and spec have dtype defined + if not isinstance(spec.dtype, list): + # spec is a simple dtype. make sure it is a subtype of the included spec dtype + if isinstance(inc_spec.dtype, list): + msg = 'Cannot extend compound data type to simple data type' + raise ValueError(msg) + if not _is_sub_dtype(spec.dtype, inc_spec.dtype): + msg = f'Cannot extend {str(inc_spec.dtype)} to {str(spec.dtype)}' + raise ValueError(msg) + return + + # spec is a compound dtype. make sure it is a subtype of the included spec dtype + if not isinstance(inc_spec.dtype, list): + msg = 'Cannot extend simple data type to compound data type' + raise ValueError(msg) + inc_spec_order = OrderedDict() + for dt in inc_spec.dtype: + inc_spec_order[dt['name']] = dt + for dt in spec.dtype: + name = dt['name'] + if name in inc_spec_order: + # verify that the extension has supplied + # a valid subtyping of existing type + inc_sub_dtype = inc_spec_order[name].dtype + new_sub_dtype = dt.dtype + if not _is_sub_dtype(new_sub_dtype, inc_sub_dtype): + msg = f'Cannot extend {str(inc_sub_dtype)} to {str(new_sub_dtype)}' + raise ValueError(msg) + # TODO do we want to disallow adding columns? (if name not in inc_spec_order) + # add/replace the new spec + inc_spec_order[name] = dt + # keep the order of the included spec + spec['dtype'] = list(inc_spec_order.values()) + +def _resolve_inc_spec_shape( + spec: Union['AttributeSpec', 'DatasetSpec'], + inc_spec: Union['AttributeSpec', 'DatasetSpec'] + ): + if inc_spec.shape is None: + # nothing to include/check + return + + if spec.shape is None: + # no shape defined, just use the included spec shape + spec['shape'] = inc_spec.shape + return + + # both inc_spec and self have shape defined + if len(spec.shape) > len(inc_spec.shape): + msg = f"Cannot extend shape {str(inc_spec.shape)} to {str(spec.shape)}" + raise ValueError(msg) + # TODO: make sure the new shape is a subset of the included shape + +def _resolve_inc_spec_dims( + spec: Union['AttributeSpec', 'DatasetSpec'], + inc_spec: Union['AttributeSpec', 'DatasetSpec'] + ): + # NOTE: In theory, the shape check above and shape & dims consistency check will catch all issues with dims + # before this function is called + if inc_spec.dims is None: + # nothing to include/check + return + + if spec.dims is None: + # no dims defined, just use the included spec dims + spec['dims'] = inc_spec.dims + return + + # both inc_spec and spec have dims defined + if len(spec.dims) > len(inc_spec.dims): # pragma: no cover + msg = f"Cannot extend dims {str(inc_spec.dims)} to {str(spec.dims)}" + raise ValueError(msg) + # TODO: make sure the new dims is a subset of the included dims + + +def _resolve_inc_spec_value( + spec: Union['AttributeSpec', 'DatasetSpec'], + inc_spec: Union['AttributeSpec', 'DatasetSpec'] + ): + # handle both default_value and value + if spec.default_value is None and inc_spec.default_value is not None: + spec['default_value'] = inc_spec.default_value + if spec.value is None and inc_spec.value is not None: + spec['value'] = inc_spec.value + + # cannot specify both value and default_value. use value if both are specified + if spec.value is not None and spec.default_value is not None: + spec['default_value'] = None + + +class ConstructableDict(dict, metaclass=ABCMeta): + @classmethod + def build_const_args(cls, spec_dict): + ''' Build constructor arguments for this ConstructableDict class from a dictionary ''' + # main use cases are when spec_dict is a ConstructableDict or a spec dict read from a file + return spec_dict.copy() + + @classmethod + def build_spec(cls, spec_dict): + ''' Build a Spec object from the given Spec dict ''' + # main use cases are when spec_dict is a ConstructableDict or a spec dict read from a file + vargs = cls.build_const_args(spec_dict) + kwargs = dict() + # iterate through the Spec docval and construct kwargs based on matching values in spec_dict + unused_vargs = list(vargs) + for x in get_docval(cls.__init__): + if x['name'] in vargs: + kwargs[x['name']] = vargs.get(x['name']) + unused_vargs.remove(x['name']) + if unused_vargs: + warn(f'Unexpected keys {unused_vargs} in spec {spec_dict}') + return cls(**kwargs) + + +class Spec(ConstructableDict): + ''' A base specification class + ''' + + @docval({'name': 'doc', 'type': str, 'doc': 'a description about what this specification represents'}, + {'name': 'name', 'type': str, 'doc': 'The name of this attribute', 'default': None}, + {'name': 'required', 'type': bool, 'doc': 'whether or not this attribute is required', 'default': True}, + {'name': 'parent', 'type': 'hdmf.spec.spec.Spec', 'doc': 'the parent of this spec', 'default': None}) + def __init__(self, **kwargs): + name, doc, required, parent = getargs('name', 'doc', 'required', 'parent', kwargs) + super().__init__() + self['doc'] = doc + if name is not None: + self['name'] = name + if not required: + self['required'] = required + self._parent = parent + + @property + def doc(self): + ''' Documentation on what this Spec is specifying ''' + return self.get('doc', None) + + @property + def name(self): + ''' The name of the object being specified ''' + return self.get('name', None) + + @property + def parent(self): + ''' The parent specification of this specification ''' + return self._parent + + @parent.setter + def parent(self, spec): + ''' Set the parent of this specification ''' + if self._parent is not None: + raise AttributeError('Cannot re-assign parent.') + self._parent = spec + + @classmethod + def build_const_args(cls, spec_dict): + ''' Build constructor arguments for this Spec class from a dictionary ''' + ret = super().build_const_args(spec_dict) + return ret + + def __hash__(self): + return id(self) + + @property + def path(self): + stack = list() + tmp = self + while tmp is not None: + name = tmp.name + if name is None: + name = tmp.data_type_def + if name is None: + name = tmp.data_type_inc + stack.append(name) + tmp = tmp.parent + return "/".join(reversed(stack)) + + +# def __eq__(self, other): +# return id(self) == id(other) + + +_target_type_key = 'target_type' + +_ref_args = [ + {'name': _target_type_key, 'type': str, 'doc': 'the target type GroupSpec or DatasetSpec'}, + {'name': 'reftype', 'type': str, + 'doc': 'the type of reference this is. only "object" is supported currently.'}, +] + + +class RefSpec(ConstructableDict): + __allowable_types = ('object', ) + + @docval(*_ref_args) + def __init__(self, **kwargs): + target_type, reftype = getargs(_target_type_key, 'reftype', kwargs) + self[_target_type_key] = target_type + if reftype not in self.__allowable_types: + msg = "reftype must be one of the following: %s" % ", ".join(self.__allowable_types) + raise ValueError(msg) + self['reftype'] = reftype + + @property + def target_type(self): + '''The data_type of the target of the reference''' + return self[_target_type_key] + + @property + def reftype(self): + '''The type of reference''' + return self['reftype'] + + +_attr_args = [ + {'name': 'name', 'type': str, 'doc': 'The name of this attribute'}, + {'name': 'doc', 'type': str, 'doc': 'a description about what this specification represents'}, + {'name': 'dtype', 'type': (str, RefSpec), 'doc': 'The data type of this attribute'}, + {'name': 'shape', 'type': (list, tuple), 'doc': 'the shape of this dataset', 'default': None}, + {'name': 'dims', 'type': (list, tuple), 'doc': 'the dimensions of this dataset', 'default': None}, + {'name': 'required', 'type': bool, + 'doc': 'whether or not this attribute is required. ignored when "value" is specified', 'default': True}, + {'name': 'parent', 'type': 'hdmf.spec.spec.BaseStorageSpec', 'doc': 'the parent of this spec', 'default': None}, + {'name': 'value', 'type': None, 'doc': 'a constant value for this attribute', 'default': None}, + {'name': 'default_value', 'type': None, 'doc': 'a default value for this attribute', 'default': None} +] + + +class AttributeSpec(Spec): + ''' Specification for attributes + ''' + + @docval(*_attr_args) + def __init__(self, **kwargs): + name, dtype, doc, dims, shape, required, parent, value, default_value = getargs( + 'name', 'dtype', 'doc', 'dims', 'shape', 'required', 'parent', 'value', 'default_value', kwargs) + super().__init__(doc, name=name, required=required, parent=parent) + self['dtype'] = DtypeHelper.check_dtype(dtype) + if value is not None: + self.pop('required', None) + self['value'] = value + if default_value is not None: + if value is not None: + raise ValueError("cannot specify 'value' and 'default_value'") + self['default_value'] = default_value + self['required'] = False + if shape is not None: + self['shape'] = shape + if dims is None: # set dummy dims "dim_0", "dim_1", ... if shape is specified but dims is not + self['dims'] = tuple(['dim_%d' % i for i in range(len(shape))]) + if dims is not None: + self['dims'] = dims + if 'shape' not in self: # set dummy shape (None, None, ...) if dims is specified but shape is not + self['shape'] = tuple([None] * len(dims)) + if self.shape is not None and self.dims is not None: + if len(self['dims']) != len(self['shape']): + raise ValueError("'dims' and 'shape' must be the same length") + + @property + def dtype(self): + ''' The data type of the attribute ''' + return self.get('dtype', None) + + @property + def value(self): + ''' The constant value of the attribute. "None" if this attribute is not constant ''' + return self.get('value', None) + + @property + def default_value(self): + ''' The default value of the attribute. "None" if this attribute has no default value ''' + return self.get('default_value', None) + + @property + def required(self): + ''' True if this attribute is required, False otherwise. ''' + return self.get('required', True) + + @property + def dims(self): + ''' The dimensions of this attribute's value ''' + return self.get('dims', None) + + @property + def shape(self): + ''' The shape of this attribute's value ''' + return self.get('shape', None) + + @classmethod + def build_const_args(cls, spec_dict): + ''' Build constructor arguments for this Spec class from a dictionary ''' + ret = super().build_const_args(spec_dict) + if isinstance(ret['dtype'], dict): + ret['dtype'] = RefSpec.build_spec(ret['dtype']) + return ret + + +_attrbl_args = [ + {'name': 'doc', 'type': str, 'doc': 'a description about what this specification represents'}, + {'name': 'name', 'type': str, + 'doc': 'the name of this base storage container, allowed only if quantity is not \'%s\' or \'%s\'' + % (ONE_OR_MANY, ZERO_OR_MANY), 'default': None}, + {'name': 'default_name', 'type': str, + 'doc': 'The default name of this base storage container, used only if name is None', 'default': None}, + {'name': 'attributes', 'type': list, 'doc': 'the attributes on this group', 'default': list()}, + {'name': 'linkable', 'type': bool, 'doc': 'whether or not this group can be linked', 'default': True}, + {'name': 'quantity', 'type': (str, int), 'doc': 'the required number of allowed instance', 'default': 1}, + {'name': 'data_type_def', 'type': str, 'doc': 'the data type this specification represents', 'default': None}, + {'name': 'data_type_inc', 'type': str, 'doc': 'the data type this specification extends', 'default': None}, +] + + +class BaseStorageSpec(Spec): + ''' A specification for any object that can hold attributes. ''' + + __inc_key = 'data_type_inc' + __def_key = 'data_type_def' + __type_key = 'data_type' + __id_key = 'object_id' + + @docval(*_attrbl_args) + def __init__(self, **kwargs): + name, doc, quantity, attributes, linkable, data_type_def, data_type_inc = \ + getargs('name', 'doc', 'quantity', 'attributes', 'linkable', 'data_type_def', 'data_type_inc', kwargs) + if name is not None and "/" in name: + raise ValueError(f"Name '{name}' is invalid. Names of Groups and Datasets cannot contain '/'") + if name is None and data_type_def is None and data_type_inc is None: + raise ValueError("Cannot create Group or Dataset spec with no name " + "without specifying '%s' and/or '%s'." % (self.def_key(), self.inc_key())) + super().__init__(doc, name=name) + default_name = getargs('default_name', kwargs) + if default_name: + if "/" in default_name: + raise ValueError( + f"Default name '{default_name}' is invalid. Names of Groups and Datasets cannot contain '/'" + ) + if name is not None: + warn("found 'default_name' with 'name' - ignoring 'default_name'") + else: + self['default_name'] = default_name + self.__attributes = dict() + if quantity in (ONE_OR_MANY, ZERO_OR_MANY): + if name is not None: + raise ValueError("Cannot give specific name to something that can " + "exist multiple times: name='%s', quantity='%s'" % (name, quantity)) + if quantity != DEF_QUANTITY: + self['quantity'] = quantity + if not linkable: + self['linkable'] = False + + if data_type_inc is not None: + if data_type_def == data_type_inc: + msg = f"data_type_inc and data_type_def cannot be the same: {data_type_inc}. Ignoring data_type_inc." + warn(msg) + else: + self[self.inc_key()] = data_type_inc + if data_type_def is not None: + self.pop('required', None) + self[self.def_key()] = data_type_def + + # self.attributes / self['attributes']: tuple/list of attributes + # self.__attributes: dict of all attributes, including attributes from parent (data_type_inc) types + # self.__new_attributes: set of attribute names that do not exist in the parent type + # self.__overridden_attributes: set of attribute names that exist in this spec and the parent type + # self.__new_attributes and self.__overridden_attributes are only set properly if resolve = True + # add all attributes described in this spec + for attribute in attributes: + self.set_attribute(attribute) + self.__new_attributes = set(self.__attributes.keys()) + self.__overridden_attributes = set() + self.__inc_spec_resolved = False + self.__resolved = False + + @property + def default_name(self): + '''The default name for this spec''' + return self.get('default_name', None) + + @property + def inc_spec_resolved(self): + return self.__inc_spec_resolved + + @property + def resolved(self): + return self.__resolved + + @resolved.setter + def resolved(self, val: bool): + if not isinstance(val, bool): + raise ValueError("resolved must be a boolean") + self.__resolved = val + + @property + def required(self): + ''' Whether or not the this spec represents a required field ''' + return self.quantity not in (ZERO_OR_ONE, ZERO_OR_MANY) + + def resolve_inc_spec(self, inc_spec: 'BaseStorageSpec', namespace: 'SpecNamespace'): + """Add attributes from the inc_spec to this spec and track which attributes are new and overridden. + + Parameters + ---------- + inc_spec : BaseStorageSpec + The BaseStorageSpec to inherit from + namespace : SpecNamespace + The namespace containing the specs - this is unused here + """ + for inc_spec_attribute in inc_spec.attributes: + self.__new_attributes.discard(inc_spec_attribute.name) + if inc_spec_attribute.name in self.__attributes: + self.__overridden_attributes.add(inc_spec_attribute.name) + new_attribute = self.__attributes[inc_spec_attribute.name] + _resolve_inc_spec_dtype(new_attribute, inc_spec_attribute) + _resolve_inc_spec_shape(new_attribute, inc_spec_attribute) + _resolve_inc_spec_dims(new_attribute, inc_spec_attribute) + _resolve_inc_spec_value(new_attribute, inc_spec_attribute) + else: + # TODO: would be nice to have inherited attributes come before new attributes in the attributes list + self.set_attribute(inc_spec_attribute) + self.__inc_spec_resolved = True + + @docval({'name': 'spec', 'type': Spec, 'doc': 'the specification to check'}) + def is_inherited_spec(self, **kwargs): + ''' + Return True if this spec was inherited from the parent type, False otherwise. + + Returns False if the spec is not found. + ''' + spec = getargs('spec', kwargs) + if spec.parent is self and spec.name in self.__attributes: + return self.is_inherited_attribute(spec.name) + return False + + @docval({'name': 'spec', 'type': Spec, 'doc': 'the specification to check'}) + def is_overridden_spec(self, **kwargs): + ''' + Return True if this spec overrides a specification from the parent type, False otherwise. + + Returns False if the spec is not found. + ''' + spec = getargs('spec', kwargs) + if spec.parent is self and spec.name in self.__attributes: + return self.is_overridden_attribute(spec.name) + return False + + @docval({'name': 'name', 'type': str, 'doc': 'the name of the attribute to check'}) + def is_inherited_attribute(self, **kwargs): + ''' + Return True if the attribute was inherited from the parent type, False otherwise. + + Raises a ValueError if the spec is not found. + ''' + name = getargs('name', kwargs) + if name not in self.__attributes: + raise ValueError("Attribute '%s' not found" % name) + return name not in self.__new_attributes + + @docval({'name': 'name', 'type': str, 'doc': 'the name of the attribute to check'}) + def is_overridden_attribute(self, **kwargs): + ''' + Return True if the given attribute overrides the specification from the parent, False otherwise. + + Raises a ValueError if the spec is not found. + ''' + name = getargs('name', kwargs) + if name not in self.__attributes: + raise ValueError("Attribute '%s' not found" % name) + return name in self.__overridden_attributes + + def is_many(self): + return self.quantity not in (1, ZERO_OR_ONE) + + @classmethod + def get_data_type_spec(cls, data_type_def): # unused + return AttributeSpec(cls.type_key(), 'the data type of this object', 'text', value=data_type_def) + + @classmethod + def get_namespace_spec(cls): # unused + return AttributeSpec('namespace', 'the namespace for the data type of this object', 'text', required=False) + + @property + def attributes(self): + ''' Tuple of attribute specifications for this specification ''' + return tuple(self.get('attributes', tuple())) + + @property + def linkable(self): + ''' True if object can be a link, False otherwise ''' + return self.get('linkable', True) + + @classmethod + def id_key(cls): + ''' Get the key used to store data ID on an instance + + Override this method to use a different name for 'object_id' + ''' + return cls.__id_key + + @classmethod + def type_key(cls): + ''' Get the key used to store data type on an instance + + Override this method to use a different name for 'data_type'. HDMF supports combining schema + that uses 'data_type' and at most one different name for 'data_type'. + ''' + return cls.__type_key + + @classmethod + def inc_key(cls): + ''' Get the key used to define a data_type include. + + Override this method to use a different keyword for 'data_type_inc'. HDMF supports combining schema + that uses 'data_type_inc' and at most one different name for 'data_type_inc'. + ''' + return cls.__inc_key + + @classmethod + def def_key(cls): + ''' Get the key used to define a data_type definition. + + Override this method to use a different keyword for 'data_type_def' HDMF supports combining schema + that uses 'data_type_def' and at most one different name for 'data_type_def'. + ''' + return cls.__def_key + + @property + def data_type_inc(self): + ''' The data type this specification inherits ''' + return self.get(self.inc_key()) + + @property + def data_type_def(self): + ''' The data type this specification defines ''' + return self.get(self.def_key(), None) + + @property + def data_type(self): + ''' The data type of this specification ''' + return self.data_type_def or self.data_type_inc + + @property + def quantity(self): + ''' The number of times the object being specified should be present ''' + return self.get('quantity', DEF_QUANTITY) + + @docval(*_attr_args) + def add_attribute(self, **kwargs): + ''' Add an attribute to this specification ''' + spec = AttributeSpec(**kwargs) + self.set_attribute(spec) + return spec + + @docval({'name': 'spec', 'type': AttributeSpec, 'doc': 'the specification for the attribute to add'}) + def set_attribute(self, **kwargs): + ''' Set an attribute on this specification ''' + spec = kwargs.get('spec') + attributes = self.setdefault('attributes', list()) + if spec.parent is not None: + spec = AttributeSpec.build_spec(spec) + # if attribute name already exists in self.__attributes, + # 1. find the attribute in self['attributes'] list and replace it with the given spec + # 2. replace the value for the name key in the self.__attributes dict + # otherwise, add the attribute spec to the self['attributes'] list and self.__attributes dict + # the values of self['attributes'] and self.__attributes should always be the same + # the former enables the spec to act like a dict with the 'attributes' key and + # the latter is useful for name-based access of attributes + if spec.name in self.__attributes: + idx = -1 + for i, attribute in enumerate(attributes): # pragma: no cover (execution should break) + if attribute.name == spec.name: + idx = i + break + if idx >= 0: + attributes[idx] = spec + else: # pragma: no cover + raise ValueError('%s in __attributes but not in spec record' % spec.name) + else: + attributes.append(spec) + self.__attributes[spec.name] = spec + spec.parent = self + + @docval({'name': 'name', 'type': str, 'doc': 'the name of the attribute to the Spec for'}) + def get_attribute(self, **kwargs): + ''' Get an attribute on this specification ''' + name = getargs('name', kwargs) + return self.__attributes.get(name) + + @classmethod + def build_const_args(cls, spec_dict): + ''' Build constructor arguments for this Spec class from a dictionary ''' + ret = super().build_const_args(spec_dict) + if 'attributes' in ret: + ret['attributes'] = [AttributeSpec.build_spec(sub_spec) for sub_spec in ret['attributes']] + return ret + + +_dt_args = [ + {'name': 'name', 'type': str, 'doc': 'the name of this column'}, + {'name': 'doc', 'type': str, 'doc': 'a description about what this data type is'}, + {'name': 'dtype', 'type': (str, list, RefSpec), 'doc': 'the data type of this column'}, +] + + +class DtypeSpec(ConstructableDict): + '''A class for specifying a component of a compound type''' + + @docval(*_dt_args) + def __init__(self, **kwargs): + doc, name, dtype = getargs('doc', 'name', 'dtype', kwargs) + self['doc'] = doc + self['name'] = name + self.check_valid_dtype(dtype) + self['dtype'] = dtype + + @property + def doc(self): + '''Documentation about this component''' + return self['doc'] + + @property + def name(self): + '''The name of this component''' + return self['name'] + + @property + def dtype(self): + ''' The data type of this component''' + return self['dtype'] + + @staticmethod + def assertValidDtype(dtype): + # Calls check_valid_dtype. This method is maintained for backwards compatibility + return DtypeSpec.check_valid_dtype(dtype) + + @staticmethod + def check_valid_dtype(dtype): + if isinstance(dtype, dict): + if _target_type_key not in dtype: + msg = "'dtype' must have the key '%s'" % _target_type_key + raise ValueError(msg) + else: + DtypeHelper.check_dtype(dtype) + return True + + @staticmethod + @docval({'name': 'spec', 'type': (str, dict), 'doc': 'the spec object to check'}, is_method=False) + def is_ref(**kwargs): + spec = getargs('spec', kwargs) + spec_is_ref = False + if isinstance(spec, dict): + if _target_type_key in spec: + spec_is_ref = True + elif 'dtype' in spec and isinstance(spec['dtype'], dict) and _target_type_key in spec['dtype']: + spec_is_ref = True + return spec_is_ref + + @classmethod + def build_const_args(cls, spec_dict): + ''' Build constructor arguments for this Spec class from a dictionary ''' + ret = super().build_const_args(spec_dict) + if isinstance(ret['dtype'], list): + ret['dtype'] = list(map(cls.build_const_args, ret['dtype'])) + elif isinstance(ret['dtype'], dict): + ret['dtype'] = RefSpec.build_spec(ret['dtype']) + return ret + + +_dataset_args = [ + {'name': 'doc', 'type': str, 'doc': 'a description about what this specification represents'}, + {'name': 'dtype', 'type': (str, list, RefSpec), + 'doc': 'The data type of this attribute. Use a list of DtypeSpecs to specify a compound data type.', + 'default': None}, + {'name': 'name', 'type': str, 'doc': 'The name of this dataset', 'default': None}, + {'name': 'default_name', 'type': str, 'doc': 'The default name of this dataset', 'default': None}, + {'name': 'shape', 'type': (list, tuple), 'doc': 'the shape of this dataset', 'default': None}, + {'name': 'dims', 'type': (list, tuple), 'doc': 'the dimensions of this dataset', 'default': None}, + {'name': 'attributes', 'type': list, 'doc': 'the attributes on this group', 'default': list()}, + {'name': 'linkable', 'type': bool, 'doc': 'whether or not this group can be linked', 'default': True}, + {'name': 'quantity', 'type': (str, int), 'doc': 'the required number of allowed instance', 'default': 1}, + {'name': 'default_value', 'type': None, 'doc': 'a default value for this dataset', 'default': None}, + {'name': 'value', 'type': None, 'doc': 'a fixed value for this dataset', 'default': None}, + {'name': 'data_type_def', 'type': str, 'doc': 'the data type this specification represents', 'default': None}, + {'name': 'data_type_inc', 'type': str, + 'doc': 'the data type this specification extends', 'default': None}, +] + + +class DatasetSpec(BaseStorageSpec): + ''' Specification for datasets + + To specify a table-like dataset i.e. a compound data type. + ''' + + @docval(*_dataset_args) + def __init__(self, **kwargs): + doc, shape, dims, dtype = popargs('doc', 'shape', 'dims', 'dtype', kwargs) + default_value, value = popargs('default_value', 'value', kwargs) + if shape is not None: + self['shape'] = shape + if dims is None: # set dummy dims "dim_0", "dim_1", ... if shape is specified but dims is not + self['dims'] = tuple(['dim_%d' % i for i in range(len(shape))]) + if dims is not None: + self['dims'] = dims + if 'shape' not in self: # set dummy shape (None, None, ...) if dims is specified but shape is not + self['shape'] = tuple([None] * len(dims)) + if self.shape is not None and self.dims is not None: + if len(self['dims']) != len(self['shape']): + raise ValueError("'dims' and 'shape' must be the same length") + if dtype is not None: + if isinstance(dtype, list): # Dtype is a compound data type + for _i, col in enumerate(dtype): + if not isinstance(col, DtypeSpec): + msg = ('must use DtypeSpec if defining compound dtype - found %s at element %d' + % (type(col), _i)) + raise ValueError(msg) + else: + DtypeHelper.check_dtype(dtype) + self['dtype'] = dtype + super().__init__(doc, **kwargs) + if default_value is not None: + self['default_value'] = default_value + if value is not None: + raise ValueError("cannot specify 'value' and 'default_value'") + if value is not None: + self['value'] = value + if self.name is not None: + valid_quant_vals = [1, 'zero_or_one', ZERO_OR_ONE] + if self.quantity not in valid_quant_vals: + raise ValueError("quantity %s invalid for spec with fixed name. Valid values are: %s" % + (self.quantity, str(valid_quant_vals))) + + + def resolve_inc_spec(self, inc_spec: 'DatasetSpec', namespace: 'SpecNamespace'): + """Add fields and attributes from the inc_spec to this spec. + + Parameters + ---------- + inc_spec : DatasetSpec + The DatasetSpec to inherit from + namespace : SpecNamespace + The namespace containing the specs - this is unused here + """ + if not isinstance(inc_spec, DatasetSpec): # TODO: replace with Pydantic type checking + raise TypeError("Cannot resolve included spec: expected DatasetSpec, got %s" % type(inc_spec)) + _resolve_inc_spec_dtype(self, inc_spec) + _resolve_inc_spec_shape(self, inc_spec) + _resolve_inc_spec_dims(self, inc_spec) + _resolve_inc_spec_value(self, inc_spec) + super().resolve_inc_spec(inc_spec, namespace) + + @property + def dims(self): + ''' The dimensions of this Dataset ''' + return self.get('dims', None) + + @property + def dtype(self): + ''' The data type of the Dataset ''' + return self.get('dtype', None) + + @property + def shape(self): + ''' The shape of the dataset ''' + return self.get('shape', None) + + @property + def default_value(self): + '''The default value of the dataset or None if not specified''' + return self.get('default_value', None) + + @property + def value(self): + '''The fixed value of the dataset or None if not specified''' + return self.get('value', None) + + @classmethod + def dtype_spec_cls(cls): + ''' The class to use when constructing DtypeSpec objects + + Override this if extending to use a class other than DtypeSpec to build + dataset specifications + ''' + return DtypeSpec + + @classmethod + def build_const_args(cls, spec_dict): + ''' Build constructor arguments for this Spec class from a dictionary ''' + ret = super().build_const_args(spec_dict) + if 'dtype' in ret: + if isinstance(ret['dtype'], list): + ret['dtype'] = list(map(cls.dtype_spec_cls().build_spec, ret['dtype'])) + elif isinstance(ret['dtype'], dict): + ret['dtype'] = RefSpec.build_spec(ret['dtype']) + return ret + + +_link_args = [ + {'name': 'doc', 'type': str, 'doc': 'a description about what this link represents'}, + {'name': _target_type_key, 'type': (str, BaseStorageSpec), 'doc': 'the target type GroupSpec or DatasetSpec'}, + {'name': 'quantity', 'type': (str, int), 'doc': 'the required number of allowed instance', 'default': 1}, + {'name': 'name', 'type': str, 'doc': 'the name of this link', 'default': None} +] + + +class LinkSpec(Spec): + + @docval(*_link_args) + def __init__(self, **kwargs): + doc, target_type, name, quantity = popargs('doc', _target_type_key, 'name', 'quantity', kwargs) + super().__init__(doc, name, **kwargs) + if isinstance(target_type, BaseStorageSpec): + if target_type.data_type_def is None: + msg = ("'%s' must be a string or a GroupSpec or DatasetSpec with a '%s' key." + % (_target_type_key, target_type.def_key())) + raise ValueError(msg) + self[_target_type_key] = target_type.data_type_def + else: + self[_target_type_key] = target_type + if quantity != 1: + self['quantity'] = quantity + + @property + def target_type(self): + ''' The data type of target specification ''' + return self.get(_target_type_key) + + @property + def data_type_inc(self): + ''' The data type of target specification ''' + return self.get(_target_type_key) + + def is_many(self): + return self.quantity not in (1, ZERO_OR_ONE) + + @property + def quantity(self): + ''' The number of times the object being specified should be present ''' + return self.get('quantity', DEF_QUANTITY) + + @property + def required(self): + ''' Whether or not the this spec represents a required field ''' + return self.quantity not in (ZERO_OR_ONE, ZERO_OR_MANY) + + +_group_args = [ + {'name': 'doc', 'type': str, 'doc': 'a description about what this specification represents'}, + { + 'name': 'name', + 'type': str, + 'doc': 'the name of the Group that is written to the file. If this argument is omitted, users will be ' + 'required to enter a ``name`` field when creating instances of this data type in the API. Another ' + 'option is to specify ``default_name``, in which case this name will be used as the name of the Group ' + 'if no other name is provided.', + 'default': None, + }, + {'name': 'default_name', 'type': str, 'doc': 'The default name of this group', 'default': None}, + {'name': 'groups', 'type': list, 'doc': 'the subgroups in this group', 'default': list()}, + {'name': 'datasets', 'type': list, 'doc': 'the datasets in this group', 'default': list()}, + {'name': 'attributes', 'type': list, 'doc': 'the attributes on this group', 'default': list()}, + {'name': 'links', 'type': list, 'doc': 'the links in this group', 'default': list()}, + {'name': 'linkable', 'type': bool, 'doc': 'whether or not this group can be linked', 'default': True}, + { + 'name': 'quantity', + 'type': (str, int), + 'doc': "the allowable number of instance of this group in a certain location. See table of options " + "`here `_. Note that if you" + "specify ``name``, ``quantity`` cannot be ``'*'``, ``'+'``, or an integer greater that 1, because you " + "cannot have more than one group of the same name in the same parent group.", + 'default': 1, + }, + {'name': 'data_type_def', 'type': str, 'doc': 'the data type this specification represents', 'default': None}, + {'name': 'data_type_inc', 'type': str, + 'doc': 'the data type this specification data_type_inc', 'default': None}, +] + + +class GroupSpec(BaseStorageSpec): + ''' Specification for groups + ''' + + @docval(*_group_args) + def __init__(self, **kwargs): + doc, groups, datasets, links = popargs('doc', 'groups', 'datasets', 'links', kwargs) + self.__data_types = dict() # for GroupSpec/DatasetSpec data_type_def/inc + self.__target_types = dict() # for LinkSpec target_types + self.__groups = dict() + for group in groups: + self.set_group(group) + self.__datasets = dict() + for dataset in datasets: + self.set_dataset(dataset) + self.__links = dict() + for link in links: + self.set_link(link) + self.__new_data_types = set(self.__data_types.keys()) + self.__new_target_types = set(self.__target_types.keys()) + self.__new_datasets = set(self.__datasets.keys()) + self.__overridden_datasets = set() + self.__new_links = set(self.__links.keys()) + self.__overridden_links = set() + self.__new_groups = set(self.__groups.keys()) + self.__overridden_groups = set() + super().__init__(doc, **kwargs) + + def resolve_inc_spec(self, inc_spec: 'GroupSpec', namespace: 'SpecNamespace'): # noqa: C901 + """Add groups, datasets, links, and attributes from the inc_spec to this spec and track which ones are new and + overridden. + + Note that data_types and target_types are not added to this spec, but are used to determine if any datasets or + links need to be added to this spec. + + Parameters + ---------- + inc_spec : GroupSpec + The GroupSpec to inherit from + namespace : SpecNamespace + The namespace containing the specs + """ + if not isinstance(inc_spec, GroupSpec): # TODO: replace with Pydantic type checking + raise TypeError("Cannot resolve included spec: expected GroupSpec, got %s" % type(inc_spec)) + data_types = list() + target_types = list() + # resolve inherited datasets + for dataset in inc_spec.datasets: + if dataset.name is None: + data_types.append(dataset) + continue + self.__new_datasets.discard(dataset.name) + if dataset.name in self.__datasets: + # check compatibility between data_type_inc of the existing dataset spec and the included dataset spec + if ( + dataset.data_type_inc != self.__datasets[dataset.name].data_type_inc and + (dataset.data_type_inc is None or self.__datasets[dataset.name].data_type_inc is None or + dataset.data_type_inc not in namespace.get_hierarchy(self.__datasets[dataset.name].data_type_inc) + ) + ): + msg = ("Cannot resolve included dataset spec '%s' with data_type_inc '%s' because a dataset " + "spec with the same name already exists with data_type_inc '%s', and data type '%s' " + "is not a child type of data type '%s'." + % (dataset.name, dataset.data_type_inc, self.__datasets[dataset.name].data_type_inc, + self.__datasets[dataset.name].data_type_inc, dataset.data_type_inc)) + raise ValueError(msg) + + # if the included dataset spec was added earlier during resolution, don't add it again + # but resolve the spec using the included dataset spec - the included spec may contain + # properties not specified in the version of this spec added earlier during resolution + self.__datasets[dataset.name].resolve_inc_spec(dataset, namespace) + self.__overridden_datasets.add(dataset.name) + else: + self.set_dataset(dataset) + # resolve inherited groups + for group in inc_spec.groups: + if group.name is None: + data_types.append(group) + continue + self.__new_groups.discard(group.name) + if group.name in self.__groups: + # check compatibility between data_type_inc of the existing group spec and the included group spec + if ( + group.data_type_inc != self.__groups[group.name].data_type_inc and + (group.data_type_inc is None or self.__groups[group.name].data_type_inc is None or + group.data_type_inc not in namespace.get_hierarchy(self.__groups[group.name].data_type_inc) + ) + ): + msg = ("Cannot resolve included group spec '%s' with data_type_inc '%s' because a group " + "spec with the same name already exists with data_type_inc '%s', and data type '%s' " + "is not a child type of data type '%s'." + % (group.name, group.data_type_inc, self.__groups[group.name].data_type_inc, + self.__groups[group.name].data_type_inc, group.data_type_inc)) + raise ValueError(msg) + + # if the included group spec was added earlier during resolution, don't add it again + # but resolve the spec using the included group spec - the included spec may contain + # properties not specified in the version of this spec added earlier during resolution + self.__groups[group.name].resolve_inc_spec(group, namespace) + self.__overridden_groups.add(group.name) + else: + self.set_group(group) + # resolve inherited links + for link in inc_spec.links: + if link.name is None: + target_types.append(link) + continue + self.__new_links.discard(link.name) + if link.name in self.__links: + # TODO: check compatibility between target_type of the existing link spec and the included link spec + self.__overridden_links.add(link.name) + else: + self.set_link(link) + # resolve inherited data_types + for dt_spec in data_types: + dt = dt_spec.data_type_def + if dt is None: + dt = dt_spec.data_type_inc + self.__new_data_types.discard(dt) + existing_dt_spec = self.get_data_type(dt) + if (existing_dt_spec is None or + ((isinstance(existing_dt_spec, list) or existing_dt_spec.name is not None) and + dt_spec.name is None)): + if isinstance(dt_spec, DatasetSpec): + self.set_dataset(dt_spec) + else: + self.set_group(dt_spec) + # resolve inherited target_types + for link_spec in target_types: + dt = link_spec.target_type + self.__new_target_types.discard(dt) + existing_dt_spec = self.get_target_type(dt) + if (existing_dt_spec is None or + (isinstance(existing_dt_spec, list) or existing_dt_spec.name is not None) and + link_spec.name is None): + self.set_link(link_spec) + super().resolve_inc_spec(inc_spec, namespace) + + @docval({'name': 'name', 'type': str, 'doc': 'the name of the dataset'}, + raises="ValueError, if 'name' is not part of this spec") + def is_inherited_dataset(self, **kwargs): + '''Return true if a dataset with the given name was inherited''' + name = getargs('name', kwargs) + if name not in self.__datasets: + raise ValueError("Dataset '%s' not found in spec" % name) + return name not in self.__new_datasets + + @docval({'name': 'name', 'type': str, 'doc': 'the name of the dataset'}, + raises="ValueError, if 'name' is not part of this spec") + def is_overridden_dataset(self, **kwargs): + '''Return true if a dataset with the given name overrides a specification from the parent type''' + name = getargs('name', kwargs) + if name not in self.__datasets: + raise ValueError("Dataset '%s' not found in spec" % name) + return name in self.__overridden_datasets + + @docval({'name': 'name', 'type': str, 'doc': 'the name of the group'}, + raises="ValueError, if 'name' is not part of this spec") + def is_inherited_group(self, **kwargs): + '''Return true if a group with the given name was inherited''' + name = getargs('name', kwargs) + if name not in self.__groups: + raise ValueError("Group '%s' not found in spec" % name) + return name not in self.__new_groups + + @docval({'name': 'name', 'type': str, 'doc': 'the name of the group'}, + raises="ValueError, if 'name' is not part of this spec") + def is_overridden_group(self, **kwargs): + '''Return true if a group with the given name overrides a specification from the parent type''' + name = getargs('name', kwargs) + if name not in self.__groups: + raise ValueError("Group '%s' not found in spec" % name) + return name in self.__overridden_groups + + @docval({'name': 'name', 'type': str, 'doc': 'the name of the link'}, + raises="ValueError, if 'name' is not part of this spec") + def is_inherited_link(self, **kwargs): + '''Return true if a link with the given name was inherited''' + name = getargs('name', kwargs) + if name not in self.__links: + raise ValueError("Link '%s' not found in spec" % name) + return name not in self.__new_links + + @docval({'name': 'name', 'type': str, 'doc': 'the name of the link'}, + raises="ValueError, if 'name' is not part of this spec") + def is_overridden_link(self, **kwargs): + '''Return true if a link with the given name overrides a specification from the parent type''' + name = getargs('name', kwargs) + if name not in self.__links: + raise ValueError("Link '%s' not found in spec" % name) + return name in self.__overridden_links + + @docval({'name': 'spec', 'type': Spec, 'doc': 'the specification to check'}) + def is_inherited_spec(self, **kwargs): + ''' Returns 'True' if specification was inherited from a parent type ''' + spec = getargs('spec', kwargs) + spec_name = spec.name + if spec_name is None and hasattr(spec, 'data_type_def'): + spec_name = spec.data_type_def + if spec_name is None: # NOTE: this will return the target type for LinkSpecs + spec_name = spec.data_type_inc + if spec_name is None: # pragma: no cover + # this should not be possible + raise ValueError('received Spec with wildcard name but no data_type_inc or data_type_def') + # if the spec has a name, it will be found in __links/__groups/__datasets before __data_types/__target_types + if spec_name in self.__links: + return self.is_inherited_link(spec_name) + elif spec_name in self.__groups: + return self.is_inherited_group(spec_name) + elif spec_name in self.__datasets: + return self.is_inherited_dataset(spec_name) + elif spec_name in self.__data_types: + # NOTE: the same data type can be both an unnamed data type and an unnamed target type + return self.is_inherited_type(spec_name) + elif spec_name in self.__target_types: + return self.is_inherited_target_type(spec_name) + elif super().is_inherited_spec(spec): # attribute spec + return True + else: + parent_name = spec.parent.name + if parent_name is None: + parent_name = spec.parent.data_type + if isinstance(spec.parent, DatasetSpec): + if (parent_name in self.__datasets and self.is_inherited_dataset(parent_name) and + self.__datasets[parent_name].get_attribute(spec_name) is not None): + return True + else: + if (parent_name in self.__groups and self.is_inherited_group(parent_name) and + self.__groups[parent_name].get_attribute(spec_name) is not None): + return True + return False + + @docval({'name': 'spec', 'type': Spec, 'doc': 'the specification to check'}) + def is_overridden_spec(self, **kwargs): + ''' Returns 'True' if specification overrides a specification from the parent type ''' + spec = getargs('spec', kwargs) + spec_name = spec.name + if spec_name is None: + if isinstance(spec, LinkSpec): # unnamed LinkSpec cannot be overridden + return False + if spec.is_many(): # this is a wildcard spec, so it cannot be overridden + return False + spec_name = spec.data_type_def + if spec_name is None: # NOTE: this will return the target type for LinkSpecs + spec_name = spec.data_type_inc + if spec_name is None: # pragma: no cover + # this should not happen + raise ValueError('received Spec with wildcard name but no data_type_inc or data_type_def') + # if the spec has a name, it will be found in __links/__groups/__datasets before __data_types/__target_types + if spec_name in self.__links: + return self.is_overridden_link(spec_name) + elif spec_name in self.__groups: + return self.is_overridden_group(spec_name) + elif spec_name in self.__datasets: + return self.is_overridden_dataset(spec_name) + elif spec_name in self.__data_types: + return self.is_overridden_type(spec_name) + elif super().is_overridden_spec(spec): # attribute spec + return True + else: + parent_name = spec.parent.name + if parent_name is None: + parent_name = spec.parent.data_type + if isinstance(spec.parent, DatasetSpec): + if (parent_name in self.__datasets and self.is_overridden_dataset(parent_name) and + self.__datasets[parent_name].is_overridden_spec(spec)): + return True + else: + if (parent_name in self.__groups and self.is_overridden_group(parent_name) and + self.__groups[parent_name].is_overridden_spec(spec)): + return True + return False + + @docval({'name': 'spec', 'type': (BaseStorageSpec, str), 'doc': 'the specification to check'}) + def is_inherited_type(self, **kwargs): + ''' Returns True if `spec` represents a data type that was inherited ''' + spec = getargs('spec', kwargs) + if isinstance(spec, BaseStorageSpec): + if spec.data_type_def is None: # why not also check data_type_inc? + raise ValueError('cannot check if something was inherited if it does not have a %s' % self.def_key()) + spec = spec.data_type_def + return spec not in self.__new_data_types + + @docval({'name': 'spec', 'type': (BaseStorageSpec, str), 'doc': 'the specification to check'}, + raises="ValueError, if 'name' is not part of this spec") + def is_overridden_type(self, **kwargs): + ''' Returns True if `spec` represents a data type that overrides a specification from a parent type ''' + return self.is_inherited_type(**kwargs) + + @docval({'name': 'spec', 'type': (LinkSpec, str), 'doc': 'the specification to check'}) + def is_inherited_target_type(self, **kwargs): + ''' Returns True if `spec` represents a target type that was inherited ''' + spec = getargs('spec', kwargs) + if isinstance(spec, LinkSpec): + spec = spec.target_type + return spec not in self.__new_target_types + + @docval({'name': 'spec', 'type': (LinkSpec, str), 'doc': 'the specification to check'}, + raises="ValueError, if 'name' is not part of this spec") + def is_overridden_target_type(self, **kwargs): + ''' Returns True if `spec` represents a target type that overrides a specification from a parent type ''' + return self.is_inherited_target_type(**kwargs) + + def __add_data_type_inc(self, spec): + # update the __data_types dict with the given groupspec/datasetspec so that: + # - if there is only one spec for a given data type, then it is stored in __data_types regardless of + # whether it is named + # - if there are multiple specs for a given data type and they are all named, then they are all stored in + # __data_types + # - if there are multiple specs for a given data type and only one is unnamed, then the unnamed spec is + # stored in __data_types + # it is not allowed to have multiple specs for a given data type and multiple are unnamed + dt = None + if hasattr(spec, 'data_type_def') and spec.data_type_def is not None: + dt = spec.data_type_def + elif hasattr(spec, 'data_type_inc') and spec.data_type_inc is not None: + dt = spec.data_type_inc + if not dt: # pragma: no cover + # this should not be possible + raise TypeError("spec does not have '%s' or '%s' defined" % (self.def_key(), self.inc_key())) + if dt in self.__data_types: + curr = self.__data_types[dt] + if curr is spec: # happens only if the same spec is added twice + return + if spec.name is None: + if isinstance(curr, list): + # replace the list of named specs associated with the data_type with this unnamed spec + # the named specs can be retrieved by name + self.__data_types[dt] = spec + else: + if curr.name is None: + # neither the spec already associated with the data_type nor the given spec have a name + msg = "Cannot have multiple groups/datasets with the same data type without specifying name" + raise ValueError(msg) + else: + # replace the named spec associated with the data_type with this unnamed spec + # the named spec can be retrieved by name + self.__data_types[dt] = spec + else: + if isinstance(curr, list): + # add this named spec to the list of named current specs associated with the data_type + self.__data_types[dt].append(spec) + else: + if curr.name is None: + # the spec associated with the data_type has no name and the given spec has a name + # leave the existing data type as is, since the new one can be retrieved by name + return + else: + # both the spec associated with the data_type and the given spec have a name + # store both specific instances of a data type + self.__data_types[dt] = [curr, spec] + else: + self.__data_types[dt] = spec + + def __add_target_type(self, spec): + # update the __target_types dict with the given linkspec so that: + # - if there is only one linkspec for a given target type, then it is stored in __target_types regardless of + # whether it is named + # - if there are multiple linkspecs for a given target type and they are all named, then they are all stored in + # __target_types + # - if there are multiple linkspecs for a given target type and only one is unnamed, then the unnamed spec is + # stored in __target_types + # it is not allowed to have multiple linkspecs for a given target type and multiple are unnamed + dt = spec.target_type + if dt in self.__target_types: + curr = self.__target_types[dt] + if curr is spec: # happens only if the same spec is added twice + return + if spec.name is None: + if isinstance(curr, list): + # replace the list of named specs associated with the target_type with this unnamed spec + # the named specs can be retrieved by name + self.__target_types[dt] = spec + else: + if curr.name is None: + # neither the spec already associated with the target_type nor the given spec have a name + msg = "Cannot have multiple links with the same target type without specifying name" + raise ValueError(msg) + else: + # replace the named spec associated with the target_type with this unnamed spec + # the named spec can be retrieved by name + self.__target_types[dt] = spec + else: + if isinstance(curr, list): + # add this named spec to the list of named current specs associated with the target_type + self.__target_types[dt].append(spec) + else: + if curr.name is None: + # the spec associated with the target_type has no name and the given spec has a name + # leave the existing data type as is, since the new one can be retrieved by name + return + else: + # both the spec associated with the target_type and the given spec have a name + # store both specific instances of a data type + self.__target_types[dt] = [curr, spec] + else: + self.__target_types[dt] = spec + + @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to retrieve'}) + def get_data_type(self, **kwargs): + ''' Get a specification by "data_type" + + NOTE: If there is only one spec for a given data type, then it is returned. + If there are multiple specs for a given data type and they are all named, then they are returned in a list. + If there are multiple specs for a given data type and only one is unnamed, then the unnamed spec is returned. + The other named specs can be returned using get_group or get_dataset. + + NOTE: this method looks for an exact match of the data type and does not consider the type hierarchy. + ''' + ndt = getargs('data_type', kwargs) + return self.__data_types.get(ndt, None) + + @docval({'name': 'target_type', 'type': str, 'doc': 'the target_type to retrieve'}) + def get_target_type(self, **kwargs): + ''' Get a specification by "target_type" + + NOTE: If there is only one spec for a given target type, then it is returned. + If there are multiple specs for a given target type and they are all named, then they are returned in a list. + If there are multiple specs for a given target type and only one is unnamed, then the unnamed spec is returned. + The other named specs can be returned using get_link. + + NOTE: this method looks for an exact match of the target type and does not consider the type hierarchy. + ''' + ndt = getargs('target_type', kwargs) + return self.__target_types.get(ndt, None) + + @property + def groups(self): + ''' The groups specified in this GroupSpec ''' + return tuple(self.get('groups', tuple())) + + @property + def datasets(self): + ''' The datasets specified in this GroupSpec ''' + return tuple(self.get('datasets', tuple())) + + @property + def links(self): + ''' The links specified in this GroupSpec ''' + return tuple(self.get('links', tuple())) + + @docval(*_group_args) + def add_group(self, **kwargs): + ''' Add a new specification for a subgroup to this group specification ''' + spec = self.__class__(**kwargs) + self.set_group(spec) + return spec + + @docval({'name': 'spec', 'type': ('GroupSpec'), 'doc': 'the specification for the subgroup'}) + def set_group(self, **kwargs): + ''' Add the given specification for a subgroup to this group specification ''' + spec = getargs('spec', kwargs) + if spec.parent is not None: + spec = self.build_spec(spec) + if spec.name is None: + if spec.data_type_inc is not None or spec.data_type_def is not None: + self.__add_data_type_inc(spec) + else: # pragma: no cover + # this should not be possible + raise TypeError("must specify 'name' or 'data_type_inc' in Group spec") + else: + # NOTE named specs can be present in both __datasets and __data_types + if spec.data_type_inc is not None or spec.data_type_def is not None: + self.__add_data_type_inc(spec) + self.__groups[spec.name] = spec + self.setdefault('groups', list()).append(spec) + spec.parent = self + + @docval({'name': 'name', 'type': str, 'doc': 'the name of the group to the Spec for'}) + def get_group(self, **kwargs): + ''' Get a specification for a subgroup to this group specification ''' + name = getargs('name', kwargs) + return self.__groups.get(name, self.__links.get(name)) + + @docval(*_dataset_args) + def add_dataset(self, **kwargs): + ''' Add a new specification for a dataset to this group specification ''' + spec = self.dataset_spec_cls()(**kwargs) + self.set_dataset(spec) + return spec + + @docval({'name': 'spec', 'type': 'hdmf.spec.spec.DatasetSpec', 'doc': 'the specification for the dataset'}) + def set_dataset(self, **kwargs): + ''' Add the given specification for a dataset to this group specification ''' + spec = getargs('spec', kwargs) + if spec.parent is not None: + spec = self.dataset_spec_cls().build_spec(spec) + if spec.name is None: + if spec.data_type_inc is not None or spec.data_type_def is not None: + self.__add_data_type_inc(spec) + else: # pragma: no cover + # this should not be possible + raise TypeError("must specify 'name' or 'data_type_inc' in Dataset spec") + else: + # NOTE named specs can be present in both __datasets and __data_types + if spec.data_type_inc is not None or spec.data_type_def is not None: + self.__add_data_type_inc(spec) + self.__datasets[spec.name] = spec + self.setdefault('datasets', list()).append(spec) + spec.parent = self + + @docval({'name': 'name', 'type': str, 'doc': 'the name of the dataset to the Spec for'}) + def get_dataset(self, **kwargs): + ''' Get a specification for a dataset to this group specification ''' + name = getargs('name', kwargs) + return self.__datasets.get(name, self.__links.get(name)) + + @docval(*_link_args) + def add_link(self, **kwargs): + ''' Add a new specification for a link to this group specification ''' + spec = self.link_spec_cls()(**kwargs) + self.set_link(spec) + return spec + + @docval({'name': 'spec', 'type': 'hdmf.spec.spec.LinkSpec', 'doc': 'the specification for the object to link to'}) + def set_link(self, **kwargs): + ''' Add a given specification for a link to this group specification ''' + spec = getargs('spec', kwargs) + if spec.parent is not None: + spec = self.link_spec_cls().build_spec(spec) + # NOTE named specs can be present in both __links and __target_types + self.__add_target_type(spec) + if spec.name is not None: + self.__links[spec.name] = spec + self.setdefault('links', list()).append(spec) + spec.parent = self + + @docval({'name': 'name', 'type': str, 'doc': 'the name of the link to the Spec for'}) + def get_link(self, **kwargs): + ''' Get a specification for a link to this group specification ''' + name = getargs('name', kwargs) + return self.__links.get(name) + + @classmethod + def dataset_spec_cls(cls): + ''' The class to use when constructing DatasetSpec objects + + Override this if extending to use a class other than DatasetSpec to build + dataset specifications + ''' + return DatasetSpec + + @classmethod + def link_spec_cls(cls): + ''' The class to use when constructing LinkSpec objects + + Override this if extending to use a class other than LinkSpec to build + link specifications + ''' + return LinkSpec + + @classmethod + def build_const_args(cls, spec_dict): + ''' Build constructor arguments for this Spec class from a dictionary ''' + ret = super().build_const_args(spec_dict) + if 'datasets' in ret: + ret['datasets'] = list(map(cls.dataset_spec_cls().build_spec, ret['datasets'])) + if 'groups' in ret: + ret['groups'] = list(map(cls.build_spec, ret['groups'])) + if 'links' in ret: + ret['links'] = list(map(cls.link_spec_cls().build_spec, ret['links'])) + return ret diff --git a/src/hdmf/spec/spec.py b/src/hdmf/spec/spec.py index 6bb761663..ebabd8e61 100644 --- a/src/hdmf/spec/spec.py +++ b/src/hdmf/spec/spec.py @@ -1,1578 +1,4 @@ -from abc import ABCMeta -from collections import OrderedDict -from copy import copy -from itertools import chain -from typing import Union, TYPE_CHECKING -from warnings import warn - -from ..utils import docval, getargs, popargs, get_docval - -if TYPE_CHECKING: - from .namespace import SpecNamespace # noqa: F401 - -NAME_WILDCARD = None # this is no longer used, but kept for backward compatibility ZERO_OR_ONE = '?' ZERO_OR_MANY = '*' ONE_OR_MANY = '+' DEF_QUANTITY = 1 -FLAGS = { - 'zero_or_one': ZERO_OR_ONE, - 'zero_or_many': ZERO_OR_MANY, - 'one_or_many': ONE_OR_MANY -} - - -class DtypeHelper: - # Dict where the keys are the primary data type and the values are list of strings with synonyms for the dtype - # make sure keys are consistent between hdmf.spec.spec.DtypeHelper.primary_dtype_synonyms, - # hdmf.build.objectmapper.ObjectMapper.__dtypes, hdmf.build.manager.TypeMap._spec_dtype_map, - # hdmf.validate.validator.__allowable, and backend dtype maps - # see https://hdmf-schema-language.readthedocs.io/en/latest/description.html#dtype - primary_dtype_synonyms = { - 'float': ["float", "float32"], - 'double': ["double", "float64"], - 'short': ["int16", "short"], - 'int': ["int32", "int"], - 'long': ["int64", "long"], - 'utf': ["text", "utf", "utf8", "utf-8"], - 'ascii': ["ascii", "bytes"], - 'bool': ["bool"], - 'int8': ["int8"], - 'uint8': ["uint8"], - 'uint16': ["uint16"], - 'uint32': ["uint32", "uint"], - 'uint64': ["uint64"], - 'object': ['object'], - 'numeric': ['numeric'], - 'isodatetime': ["isodatetime", "datetime", "date"] - } - - # List of recommended primary dtype strings. These are the keys of primary_dtype_string_synonyms - recommended_primary_dtypes = list(primary_dtype_synonyms.keys()) - - # List of valid primary data type strings - valid_primary_dtypes = set(list(primary_dtype_synonyms.keys()) + - [vi for v in primary_dtype_synonyms.values() for vi in v]) - - @staticmethod - def simplify_cpd_type(cpd_type): - ''' - Transform a list of DtypeSpecs into a list of strings. - Use for simple representation of compound type and - validation. - - :param cpd_type: The list of DtypeSpecs to simplify - :type cpd_type: list - - ''' - ret = list() - for exp in cpd_type: - exp_key = exp.dtype - if isinstance(exp_key, RefSpec): - exp_key = exp_key.reftype - ret.append(exp_key) - return ret - - @staticmethod - def check_dtype(dtype): - """Check that the dtype string is a reference or a valid primary dtype.""" - if not isinstance(dtype, RefSpec) and dtype not in DtypeHelper.valid_primary_dtypes: - raise ValueError("dtype '%s' is not a valid primary data type. Allowed dtypes: %s" - % (dtype, str(DtypeHelper.valid_primary_dtypes))) - return dtype - - # all keys and values should be keys in primary_dtype_synonyms - additional_allowed = { - 'float': ['double'], - 'int8': ['short', 'int', 'long'], - 'short': ['int', 'long'], - 'int': ['long'], - 'uint8': ['uint16', 'uint32', 'uint64'], - 'uint16': ['uint32', 'uint64'], - 'uint32': ['uint64'], - 'utf': ['ascii'] - } - - # if the spec dtype is a key in __allowable, then all types in __allowable[key] are valid - allowable = dict() - for dt, dt_syn in primary_dtype_synonyms.items(): - allow = copy(dt_syn) - if dt in additional_allowed: - for addl in additional_allowed[dt]: - allow.extend(primary_dtype_synonyms[addl]) - for syn in dt_syn: - allowable[syn] = allow - allowable['numeric'].extend(set(chain.from_iterable(v for k, v in allowable.items() if 'int' in k or 'float' in k))) - - @staticmethod - def is_allowed_dtype(new: str, orig: str): - if orig not in DtypeHelper.allowable: - raise ValueError(f"Unknown dtype '{orig}'") - return new in DtypeHelper.allowable[orig] - - -def _is_sub_dtype(new: Union[str, "RefSpec"], orig: Union[str, "RefSpec"]): - if isinstance(orig, RefSpec) != isinstance(new, RefSpec): - return False - - if isinstance(orig, RefSpec): # both are RefSpec - # check ref target is a subtype of the original ref target - # TODO: implement subtype check for RefSpec. might need to resolve RefSpec target type to a spec first - # return orig == new - return True - else: - return DtypeHelper.is_allowed_dtype(new, orig) - - -def _resolve_inc_spec_dtype( - spec: Union['AttributeSpec', 'DatasetSpec'], - inc_spec: Union['AttributeSpec', 'DatasetSpec'] - ): - if inc_spec.dtype is None: - # nothing to include/check - return - - if spec.dtype is None: - # no dtype defined, just use the included spec dtype - spec['dtype'] = inc_spec.dtype - return - - # both inc_spec and spec have dtype defined - if not isinstance(spec.dtype, list): - # spec is a simple dtype. make sure it is a subtype of the included spec dtype - if isinstance(inc_spec.dtype, list): - msg = 'Cannot extend compound data type to simple data type' - raise ValueError(msg) - if not _is_sub_dtype(spec.dtype, inc_spec.dtype): - msg = f'Cannot extend {str(inc_spec.dtype)} to {str(spec.dtype)}' - raise ValueError(msg) - return - - # spec is a compound dtype. make sure it is a subtype of the included spec dtype - if not isinstance(inc_spec.dtype, list): - msg = 'Cannot extend simple data type to compound data type' - raise ValueError(msg) - inc_spec_order = OrderedDict() - for dt in inc_spec.dtype: - inc_spec_order[dt['name']] = dt - for dt in spec.dtype: - name = dt['name'] - if name in inc_spec_order: - # verify that the extension has supplied - # a valid subtyping of existing type - inc_sub_dtype = inc_spec_order[name].dtype - new_sub_dtype = dt.dtype - if not _is_sub_dtype(new_sub_dtype, inc_sub_dtype): - msg = f'Cannot extend {str(inc_sub_dtype)} to {str(new_sub_dtype)}' - raise ValueError(msg) - # TODO do we want to disallow adding columns? (if name not in inc_spec_order) - # add/replace the new spec - inc_spec_order[name] = dt - # keep the order of the included spec - spec['dtype'] = list(inc_spec_order.values()) - -def _resolve_inc_spec_shape( - spec: Union['AttributeSpec', 'DatasetSpec'], - inc_spec: Union['AttributeSpec', 'DatasetSpec'] - ): - if inc_spec.shape is None: - # nothing to include/check - return - - if spec.shape is None: - # no shape defined, just use the included spec shape - spec['shape'] = inc_spec.shape - return - - # both inc_spec and self have shape defined - if len(spec.shape) > len(inc_spec.shape): - msg = f"Cannot extend shape {str(inc_spec.shape)} to {str(spec.shape)}" - raise ValueError(msg) - # TODO: make sure the new shape is a subset of the included shape - -def _resolve_inc_spec_dims( - spec: Union['AttributeSpec', 'DatasetSpec'], - inc_spec: Union['AttributeSpec', 'DatasetSpec'] - ): - # NOTE: In theory, the shape check above and shape & dims consistency check will catch all issues with dims - # before this function is called - if inc_spec.dims is None: - # nothing to include/check - return - - if spec.dims is None: - # no dims defined, just use the included spec dims - spec['dims'] = inc_spec.dims - return - - # both inc_spec and spec have dims defined - if len(spec.dims) > len(inc_spec.dims): # pragma: no cover - msg = f"Cannot extend dims {str(inc_spec.dims)} to {str(spec.dims)}" - raise ValueError(msg) - # TODO: make sure the new dims is a subset of the included dims - - -def _resolve_inc_spec_value( - spec: Union['AttributeSpec', 'DatasetSpec'], - inc_spec: Union['AttributeSpec', 'DatasetSpec'] - ): - # handle both default_value and value - if spec.default_value is None and inc_spec.default_value is not None: - spec['default_value'] = inc_spec.default_value - if spec.value is None and inc_spec.value is not None: - spec['value'] = inc_spec.value - - # cannot specify both value and default_value. use value if both are specified - if spec.value is not None and spec.default_value is not None: - spec['default_value'] = None - - -class ConstructableDict(dict, metaclass=ABCMeta): - @classmethod - def build_const_args(cls, spec_dict): - ''' Build constructor arguments for this ConstructableDict class from a dictionary ''' - # main use cases are when spec_dict is a ConstructableDict or a spec dict read from a file - return spec_dict.copy() - - @classmethod - def build_spec(cls, spec_dict): - ''' Build a Spec object from the given Spec dict ''' - # main use cases are when spec_dict is a ConstructableDict or a spec dict read from a file - vargs = cls.build_const_args(spec_dict) - kwargs = dict() - # iterate through the Spec docval and construct kwargs based on matching values in spec_dict - unused_vargs = list(vargs) - for x in get_docval(cls.__init__): - if x['name'] in vargs: - kwargs[x['name']] = vargs.get(x['name']) - unused_vargs.remove(x['name']) - if unused_vargs: - warn(f'Unexpected keys {unused_vargs} in spec {spec_dict}') - return cls(**kwargs) - - -class Spec(ConstructableDict): - ''' A base specification class - ''' - - @docval({'name': 'doc', 'type': str, 'doc': 'a description about what this specification represents'}, - {'name': 'name', 'type': str, 'doc': 'The name of this attribute', 'default': None}, - {'name': 'required', 'type': bool, 'doc': 'whether or not this attribute is required', 'default': True}, - {'name': 'parent', 'type': 'hdmf.spec.spec.Spec', 'doc': 'the parent of this spec', 'default': None}) - def __init__(self, **kwargs): - name, doc, required, parent = getargs('name', 'doc', 'required', 'parent', kwargs) - super().__init__() - self['doc'] = doc - if name is not None: - self['name'] = name - if not required: - self['required'] = required - self._parent = parent - - @property - def doc(self): - ''' Documentation on what this Spec is specifying ''' - return self.get('doc', None) - - @property - def name(self): - ''' The name of the object being specified ''' - return self.get('name', None) - - @property - def parent(self): - ''' The parent specification of this specification ''' - return self._parent - - @parent.setter - def parent(self, spec): - ''' Set the parent of this specification ''' - if self._parent is not None: - raise AttributeError('Cannot re-assign parent.') - self._parent = spec - - @classmethod - def build_const_args(cls, spec_dict): - ''' Build constructor arguments for this Spec class from a dictionary ''' - ret = super().build_const_args(spec_dict) - return ret - - def __hash__(self): - return id(self) - - @property - def path(self): - stack = list() - tmp = self - while tmp is not None: - name = tmp.name - if name is None: - name = tmp.data_type_def - if name is None: - name = tmp.data_type_inc - stack.append(name) - tmp = tmp.parent - return "/".join(reversed(stack)) - - -# def __eq__(self, other): -# return id(self) == id(other) - - -_target_type_key = 'target_type' - -_ref_args = [ - {'name': _target_type_key, 'type': str, 'doc': 'the target type GroupSpec or DatasetSpec'}, - {'name': 'reftype', 'type': str, - 'doc': 'the type of reference this is. only "object" is supported currently.'}, -] - - -class RefSpec(ConstructableDict): - __allowable_types = ('object', ) - - @docval(*_ref_args) - def __init__(self, **kwargs): - target_type, reftype = getargs(_target_type_key, 'reftype', kwargs) - self[_target_type_key] = target_type - if reftype not in self.__allowable_types: - msg = "reftype must be one of the following: %s" % ", ".join(self.__allowable_types) - raise ValueError(msg) - self['reftype'] = reftype - - @property - def target_type(self): - '''The data_type of the target of the reference''' - return self[_target_type_key] - - @property - def reftype(self): - '''The type of reference''' - return self['reftype'] - - -_attr_args = [ - {'name': 'name', 'type': str, 'doc': 'The name of this attribute'}, - {'name': 'doc', 'type': str, 'doc': 'a description about what this specification represents'}, - {'name': 'dtype', 'type': (str, RefSpec), 'doc': 'The data type of this attribute'}, - {'name': 'shape', 'type': (list, tuple), 'doc': 'the shape of this dataset', 'default': None}, - {'name': 'dims', 'type': (list, tuple), 'doc': 'the dimensions of this dataset', 'default': None}, - {'name': 'required', 'type': bool, - 'doc': 'whether or not this attribute is required. ignored when "value" is specified', 'default': True}, - {'name': 'parent', 'type': 'hdmf.spec.spec.BaseStorageSpec', 'doc': 'the parent of this spec', 'default': None}, - {'name': 'value', 'type': None, 'doc': 'a constant value for this attribute', 'default': None}, - {'name': 'default_value', 'type': None, 'doc': 'a default value for this attribute', 'default': None} -] - - -class AttributeSpec(Spec): - ''' Specification for attributes - ''' - - @docval(*_attr_args) - def __init__(self, **kwargs): - name, dtype, doc, dims, shape, required, parent, value, default_value = getargs( - 'name', 'dtype', 'doc', 'dims', 'shape', 'required', 'parent', 'value', 'default_value', kwargs) - super().__init__(doc, name=name, required=required, parent=parent) - self['dtype'] = DtypeHelper.check_dtype(dtype) - if value is not None: - self.pop('required', None) - self['value'] = value - if default_value is not None: - if value is not None: - raise ValueError("cannot specify 'value' and 'default_value'") - self['default_value'] = default_value - self['required'] = False - if shape is not None: - self['shape'] = shape - if dims is None: # set dummy dims "dim_0", "dim_1", ... if shape is specified but dims is not - self['dims'] = tuple(['dim_%d' % i for i in range(len(shape))]) - if dims is not None: - self['dims'] = dims - if 'shape' not in self: # set dummy shape (None, None, ...) if dims is specified but shape is not - self['shape'] = tuple([None] * len(dims)) - if self.shape is not None and self.dims is not None: - if len(self['dims']) != len(self['shape']): - raise ValueError("'dims' and 'shape' must be the same length") - - @property - def dtype(self): - ''' The data type of the attribute ''' - return self.get('dtype', None) - - @property - def value(self): - ''' The constant value of the attribute. "None" if this attribute is not constant ''' - return self.get('value', None) - - @property - def default_value(self): - ''' The default value of the attribute. "None" if this attribute has no default value ''' - return self.get('default_value', None) - - @property - def required(self): - ''' True if this attribute is required, False otherwise. ''' - return self.get('required', True) - - @property - def dims(self): - ''' The dimensions of this attribute's value ''' - return self.get('dims', None) - - @property - def shape(self): - ''' The shape of this attribute's value ''' - return self.get('shape', None) - - @classmethod - def build_const_args(cls, spec_dict): - ''' Build constructor arguments for this Spec class from a dictionary ''' - ret = super().build_const_args(spec_dict) - if isinstance(ret['dtype'], dict): - ret['dtype'] = RefSpec.build_spec(ret['dtype']) - return ret - - -_attrbl_args = [ - {'name': 'doc', 'type': str, 'doc': 'a description about what this specification represents'}, - {'name': 'name', 'type': str, - 'doc': 'the name of this base storage container, allowed only if quantity is not \'%s\' or \'%s\'' - % (ONE_OR_MANY, ZERO_OR_MANY), 'default': None}, - {'name': 'default_name', 'type': str, - 'doc': 'The default name of this base storage container, used only if name is None', 'default': None}, - {'name': 'attributes', 'type': list, 'doc': 'the attributes on this group', 'default': list()}, - {'name': 'linkable', 'type': bool, 'doc': 'whether or not this group can be linked', 'default': True}, - {'name': 'quantity', 'type': (str, int), 'doc': 'the required number of allowed instance', 'default': 1}, - {'name': 'data_type_def', 'type': str, 'doc': 'the data type this specification represents', 'default': None}, - {'name': 'data_type_inc', 'type': str, 'doc': 'the data type this specification extends', 'default': None}, -] - - -class BaseStorageSpec(Spec): - ''' A specification for any object that can hold attributes. ''' - - __inc_key = 'data_type_inc' - __def_key = 'data_type_def' - __type_key = 'data_type' - __id_key = 'object_id' - - @docval(*_attrbl_args) - def __init__(self, **kwargs): - name, doc, quantity, attributes, linkable, data_type_def, data_type_inc = \ - getargs('name', 'doc', 'quantity', 'attributes', 'linkable', 'data_type_def', 'data_type_inc', kwargs) - if name is not None and "/" in name: - raise ValueError(f"Name '{name}' is invalid. Names of Groups and Datasets cannot contain '/'") - if name is None and data_type_def is None and data_type_inc is None: - raise ValueError("Cannot create Group or Dataset spec with no name " - "without specifying '%s' and/or '%s'." % (self.def_key(), self.inc_key())) - super().__init__(doc, name=name) - default_name = getargs('default_name', kwargs) - if default_name: - if "/" in default_name: - raise ValueError( - f"Default name '{default_name}' is invalid. Names of Groups and Datasets cannot contain '/'" - ) - if name is not None: - warn("found 'default_name' with 'name' - ignoring 'default_name'") - else: - self['default_name'] = default_name - self.__attributes = dict() - if quantity in (ONE_OR_MANY, ZERO_OR_MANY): - if name is not None: - raise ValueError("Cannot give specific name to something that can " - "exist multiple times: name='%s', quantity='%s'" % (name, quantity)) - if quantity != DEF_QUANTITY: - self['quantity'] = quantity - if not linkable: - self['linkable'] = False - - if data_type_inc is not None: - if data_type_def == data_type_inc: - msg = f"data_type_inc and data_type_def cannot be the same: {data_type_inc}. Ignoring data_type_inc." - warn(msg) - else: - self[self.inc_key()] = data_type_inc - if data_type_def is not None: - self.pop('required', None) - self[self.def_key()] = data_type_def - - # self.attributes / self['attributes']: tuple/list of attributes - # self.__attributes: dict of all attributes, including attributes from parent (data_type_inc) types - # self.__new_attributes: set of attribute names that do not exist in the parent type - # self.__overridden_attributes: set of attribute names that exist in this spec and the parent type - # self.__new_attributes and self.__overridden_attributes are only set properly if resolve = True - # add all attributes described in this spec - for attribute in attributes: - self.set_attribute(attribute) - self.__new_attributes = set(self.__attributes.keys()) - self.__overridden_attributes = set() - self.__inc_spec_resolved = False - self.__resolved = False - - @property - def default_name(self): - '''The default name for this spec''' - return self.get('default_name', None) - - @property - def inc_spec_resolved(self): - return self.__inc_spec_resolved - - @property - def resolved(self): - return self.__resolved - - @resolved.setter - def resolved(self, val: bool): - if not isinstance(val, bool): - raise ValueError("resolved must be a boolean") - self.__resolved = val - - @property - def required(self): - ''' Whether or not the this spec represents a required field ''' - return self.quantity not in (ZERO_OR_ONE, ZERO_OR_MANY) - - def resolve_inc_spec(self, inc_spec: 'BaseStorageSpec', namespace: 'SpecNamespace'): - """Add attributes from the inc_spec to this spec and track which attributes are new and overridden. - - Parameters - ---------- - inc_spec : BaseStorageSpec - The BaseStorageSpec to inherit from - namespace : SpecNamespace - The namespace containing the specs - this is unused here - """ - for inc_spec_attribute in inc_spec.attributes: - self.__new_attributes.discard(inc_spec_attribute.name) - if inc_spec_attribute.name in self.__attributes: - self.__overridden_attributes.add(inc_spec_attribute.name) - new_attribute = self.__attributes[inc_spec_attribute.name] - _resolve_inc_spec_dtype(new_attribute, inc_spec_attribute) - _resolve_inc_spec_shape(new_attribute, inc_spec_attribute) - _resolve_inc_spec_dims(new_attribute, inc_spec_attribute) - _resolve_inc_spec_value(new_attribute, inc_spec_attribute) - else: - # TODO: would be nice to have inherited attributes come before new attributes in the attributes list - self.set_attribute(inc_spec_attribute) - self.__inc_spec_resolved = True - - @docval({'name': 'spec', 'type': Spec, 'doc': 'the specification to check'}) - def is_inherited_spec(self, **kwargs): - ''' - Return True if this spec was inherited from the parent type, False otherwise. - - Returns False if the spec is not found. - ''' - spec = getargs('spec', kwargs) - if spec.parent is self and spec.name in self.__attributes: - return self.is_inherited_attribute(spec.name) - return False - - @docval({'name': 'spec', 'type': Spec, 'doc': 'the specification to check'}) - def is_overridden_spec(self, **kwargs): - ''' - Return True if this spec overrides a specification from the parent type, False otherwise. - - Returns False if the spec is not found. - ''' - spec = getargs('spec', kwargs) - if spec.parent is self and spec.name in self.__attributes: - return self.is_overridden_attribute(spec.name) - return False - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the attribute to check'}) - def is_inherited_attribute(self, **kwargs): - ''' - Return True if the attribute was inherited from the parent type, False otherwise. - - Raises a ValueError if the spec is not found. - ''' - name = getargs('name', kwargs) - if name not in self.__attributes: - raise ValueError("Attribute '%s' not found" % name) - return name not in self.__new_attributes - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the attribute to check'}) - def is_overridden_attribute(self, **kwargs): - ''' - Return True if the given attribute overrides the specification from the parent, False otherwise. - - Raises a ValueError if the spec is not found. - ''' - name = getargs('name', kwargs) - if name not in self.__attributes: - raise ValueError("Attribute '%s' not found" % name) - return name in self.__overridden_attributes - - def is_many(self): - return self.quantity not in (1, ZERO_OR_ONE) - - @classmethod - def get_data_type_spec(cls, data_type_def): # unused - return AttributeSpec(cls.type_key(), 'the data type of this object', 'text', value=data_type_def) - - @classmethod - def get_namespace_spec(cls): # unused - return AttributeSpec('namespace', 'the namespace for the data type of this object', 'text', required=False) - - @property - def attributes(self): - ''' Tuple of attribute specifications for this specification ''' - return tuple(self.get('attributes', tuple())) - - @property - def linkable(self): - ''' True if object can be a link, False otherwise ''' - return self.get('linkable', True) - - @classmethod - def id_key(cls): - ''' Get the key used to store data ID on an instance - - Override this method to use a different name for 'object_id' - ''' - return cls.__id_key - - @classmethod - def type_key(cls): - ''' Get the key used to store data type on an instance - - Override this method to use a different name for 'data_type'. HDMF supports combining schema - that uses 'data_type' and at most one different name for 'data_type'. - ''' - return cls.__type_key - - @classmethod - def inc_key(cls): - ''' Get the key used to define a data_type include. - - Override this method to use a different keyword for 'data_type_inc'. HDMF supports combining schema - that uses 'data_type_inc' and at most one different name for 'data_type_inc'. - ''' - return cls.__inc_key - - @classmethod - def def_key(cls): - ''' Get the key used to define a data_type definition. - - Override this method to use a different keyword for 'data_type_def' HDMF supports combining schema - that uses 'data_type_def' and at most one different name for 'data_type_def'. - ''' - return cls.__def_key - - @property - def data_type_inc(self): - ''' The data type this specification inherits ''' - return self.get(self.inc_key()) - - @property - def data_type_def(self): - ''' The data type this specification defines ''' - return self.get(self.def_key(), None) - - @property - def data_type(self): - ''' The data type of this specification ''' - return self.data_type_def or self.data_type_inc - - @property - def quantity(self): - ''' The number of times the object being specified should be present ''' - return self.get('quantity', DEF_QUANTITY) - - @docval(*_attr_args) - def add_attribute(self, **kwargs): - ''' Add an attribute to this specification ''' - spec = AttributeSpec(**kwargs) - self.set_attribute(spec) - return spec - - @docval({'name': 'spec', 'type': AttributeSpec, 'doc': 'the specification for the attribute to add'}) - def set_attribute(self, **kwargs): - ''' Set an attribute on this specification ''' - spec = kwargs.get('spec') - attributes = self.setdefault('attributes', list()) - if spec.parent is not None: - spec = AttributeSpec.build_spec(spec) - # if attribute name already exists in self.__attributes, - # 1. find the attribute in self['attributes'] list and replace it with the given spec - # 2. replace the value for the name key in the self.__attributes dict - # otherwise, add the attribute spec to the self['attributes'] list and self.__attributes dict - # the values of self['attributes'] and self.__attributes should always be the same - # the former enables the spec to act like a dict with the 'attributes' key and - # the latter is useful for name-based access of attributes - if spec.name in self.__attributes: - idx = -1 - for i, attribute in enumerate(attributes): # pragma: no cover (execution should break) - if attribute.name == spec.name: - idx = i - break - if idx >= 0: - attributes[idx] = spec - else: # pragma: no cover - raise ValueError('%s in __attributes but not in spec record' % spec.name) - else: - attributes.append(spec) - self.__attributes[spec.name] = spec - spec.parent = self - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the attribute to the Spec for'}) - def get_attribute(self, **kwargs): - ''' Get an attribute on this specification ''' - name = getargs('name', kwargs) - return self.__attributes.get(name) - - @classmethod - def build_const_args(cls, spec_dict): - ''' Build constructor arguments for this Spec class from a dictionary ''' - ret = super().build_const_args(spec_dict) - if 'attributes' in ret: - ret['attributes'] = [AttributeSpec.build_spec(sub_spec) for sub_spec in ret['attributes']] - return ret - - -_dt_args = [ - {'name': 'name', 'type': str, 'doc': 'the name of this column'}, - {'name': 'doc', 'type': str, 'doc': 'a description about what this data type is'}, - {'name': 'dtype', 'type': (str, list, RefSpec), 'doc': 'the data type of this column'}, -] - - -class DtypeSpec(ConstructableDict): - '''A class for specifying a component of a compound type''' - - @docval(*_dt_args) - def __init__(self, **kwargs): - doc, name, dtype = getargs('doc', 'name', 'dtype', kwargs) - self['doc'] = doc - self['name'] = name - self.check_valid_dtype(dtype) - self['dtype'] = dtype - - @property - def doc(self): - '''Documentation about this component''' - return self['doc'] - - @property - def name(self): - '''The name of this component''' - return self['name'] - - @property - def dtype(self): - ''' The data type of this component''' - return self['dtype'] - - @staticmethod - def assertValidDtype(dtype): - # Calls check_valid_dtype. This method is maintained for backwards compatibility - return DtypeSpec.check_valid_dtype(dtype) - - @staticmethod - def check_valid_dtype(dtype): - if isinstance(dtype, dict): - if _target_type_key not in dtype: - msg = "'dtype' must have the key '%s'" % _target_type_key - raise ValueError(msg) - else: - DtypeHelper.check_dtype(dtype) - return True - - @staticmethod - @docval({'name': 'spec', 'type': (str, dict), 'doc': 'the spec object to check'}, is_method=False) - def is_ref(**kwargs): - spec = getargs('spec', kwargs) - spec_is_ref = False - if isinstance(spec, dict): - if _target_type_key in spec: - spec_is_ref = True - elif 'dtype' in spec and isinstance(spec['dtype'], dict) and _target_type_key in spec['dtype']: - spec_is_ref = True - return spec_is_ref - - @classmethod - def build_const_args(cls, spec_dict): - ''' Build constructor arguments for this Spec class from a dictionary ''' - ret = super().build_const_args(spec_dict) - if isinstance(ret['dtype'], list): - ret['dtype'] = list(map(cls.build_const_args, ret['dtype'])) - elif isinstance(ret['dtype'], dict): - ret['dtype'] = RefSpec.build_spec(ret['dtype']) - return ret - - -_dataset_args = [ - {'name': 'doc', 'type': str, 'doc': 'a description about what this specification represents'}, - {'name': 'dtype', 'type': (str, list, RefSpec), - 'doc': 'The data type of this attribute. Use a list of DtypeSpecs to specify a compound data type.', - 'default': None}, - {'name': 'name', 'type': str, 'doc': 'The name of this dataset', 'default': None}, - {'name': 'default_name', 'type': str, 'doc': 'The default name of this dataset', 'default': None}, - {'name': 'shape', 'type': (list, tuple), 'doc': 'the shape of this dataset', 'default': None}, - {'name': 'dims', 'type': (list, tuple), 'doc': 'the dimensions of this dataset', 'default': None}, - {'name': 'attributes', 'type': list, 'doc': 'the attributes on this group', 'default': list()}, - {'name': 'linkable', 'type': bool, 'doc': 'whether or not this group can be linked', 'default': True}, - {'name': 'quantity', 'type': (str, int), 'doc': 'the required number of allowed instance', 'default': 1}, - {'name': 'default_value', 'type': None, 'doc': 'a default value for this dataset', 'default': None}, - {'name': 'value', 'type': None, 'doc': 'a fixed value for this dataset', 'default': None}, - {'name': 'data_type_def', 'type': str, 'doc': 'the data type this specification represents', 'default': None}, - {'name': 'data_type_inc', 'type': str, - 'doc': 'the data type this specification extends', 'default': None}, -] - - -class DatasetSpec(BaseStorageSpec): - ''' Specification for datasets - - To specify a table-like dataset i.e. a compound data type. - ''' - - @docval(*_dataset_args) - def __init__(self, **kwargs): - doc, shape, dims, dtype = popargs('doc', 'shape', 'dims', 'dtype', kwargs) - default_value, value = popargs('default_value', 'value', kwargs) - if shape is not None: - self['shape'] = shape - if dims is None: # set dummy dims "dim_0", "dim_1", ... if shape is specified but dims is not - self['dims'] = tuple(['dim_%d' % i for i in range(len(shape))]) - if dims is not None: - self['dims'] = dims - if 'shape' not in self: # set dummy shape (None, None, ...) if dims is specified but shape is not - self['shape'] = tuple([None] * len(dims)) - if self.shape is not None and self.dims is not None: - if len(self['dims']) != len(self['shape']): - raise ValueError("'dims' and 'shape' must be the same length") - if dtype is not None: - if isinstance(dtype, list): # Dtype is a compound data type - for _i, col in enumerate(dtype): - if not isinstance(col, DtypeSpec): - msg = ('must use DtypeSpec if defining compound dtype - found %s at element %d' - % (type(col), _i)) - raise ValueError(msg) - else: - DtypeHelper.check_dtype(dtype) - self['dtype'] = dtype - super().__init__(doc, **kwargs) - if default_value is not None: - self['default_value'] = default_value - if value is not None: - raise ValueError("cannot specify 'value' and 'default_value'") - if value is not None: - self['value'] = value - if self.name is not None: - valid_quant_vals = [1, 'zero_or_one', ZERO_OR_ONE] - if self.quantity not in valid_quant_vals: - raise ValueError("quantity %s invalid for spec with fixed name. Valid values are: %s" % - (self.quantity, str(valid_quant_vals))) - - - def resolve_inc_spec(self, inc_spec: 'DatasetSpec', namespace: 'SpecNamespace'): - """Add fields and attributes from the inc_spec to this spec. - - Parameters - ---------- - inc_spec : DatasetSpec - The DatasetSpec to inherit from - namespace : SpecNamespace - The namespace containing the specs - this is unused here - """ - if not isinstance(inc_spec, DatasetSpec): # TODO: replace with Pydantic type checking - raise TypeError("Cannot resolve included spec: expected DatasetSpec, got %s" % type(inc_spec)) - _resolve_inc_spec_dtype(self, inc_spec) - _resolve_inc_spec_shape(self, inc_spec) - _resolve_inc_spec_dims(self, inc_spec) - _resolve_inc_spec_value(self, inc_spec) - super().resolve_inc_spec(inc_spec, namespace) - - @property - def dims(self): - ''' The dimensions of this Dataset ''' - return self.get('dims', None) - - @property - def dtype(self): - ''' The data type of the Dataset ''' - return self.get('dtype', None) - - @property - def shape(self): - ''' The shape of the dataset ''' - return self.get('shape', None) - - @property - def default_value(self): - '''The default value of the dataset or None if not specified''' - return self.get('default_value', None) - - @property - def value(self): - '''The fixed value of the dataset or None if not specified''' - return self.get('value', None) - - @classmethod - def dtype_spec_cls(cls): - ''' The class to use when constructing DtypeSpec objects - - Override this if extending to use a class other than DtypeSpec to build - dataset specifications - ''' - return DtypeSpec - - @classmethod - def build_const_args(cls, spec_dict): - ''' Build constructor arguments for this Spec class from a dictionary ''' - ret = super().build_const_args(spec_dict) - if 'dtype' in ret: - if isinstance(ret['dtype'], list): - ret['dtype'] = list(map(cls.dtype_spec_cls().build_spec, ret['dtype'])) - elif isinstance(ret['dtype'], dict): - ret['dtype'] = RefSpec.build_spec(ret['dtype']) - return ret - - -_link_args = [ - {'name': 'doc', 'type': str, 'doc': 'a description about what this link represents'}, - {'name': _target_type_key, 'type': (str, BaseStorageSpec), 'doc': 'the target type GroupSpec or DatasetSpec'}, - {'name': 'quantity', 'type': (str, int), 'doc': 'the required number of allowed instance', 'default': 1}, - {'name': 'name', 'type': str, 'doc': 'the name of this link', 'default': None} -] - - -class LinkSpec(Spec): - - @docval(*_link_args) - def __init__(self, **kwargs): - doc, target_type, name, quantity = popargs('doc', _target_type_key, 'name', 'quantity', kwargs) - super().__init__(doc, name, **kwargs) - if isinstance(target_type, BaseStorageSpec): - if target_type.data_type_def is None: - msg = ("'%s' must be a string or a GroupSpec or DatasetSpec with a '%s' key." - % (_target_type_key, target_type.def_key())) - raise ValueError(msg) - self[_target_type_key] = target_type.data_type_def - else: - self[_target_type_key] = target_type - if quantity != 1: - self['quantity'] = quantity - - @property - def target_type(self): - ''' The data type of target specification ''' - return self.get(_target_type_key) - - @property - def data_type_inc(self): - ''' The data type of target specification ''' - return self.get(_target_type_key) - - def is_many(self): - return self.quantity not in (1, ZERO_OR_ONE) - - @property - def quantity(self): - ''' The number of times the object being specified should be present ''' - return self.get('quantity', DEF_QUANTITY) - - @property - def required(self): - ''' Whether or not the this spec represents a required field ''' - return self.quantity not in (ZERO_OR_ONE, ZERO_OR_MANY) - - -_group_args = [ - {'name': 'doc', 'type': str, 'doc': 'a description about what this specification represents'}, - { - 'name': 'name', - 'type': str, - 'doc': 'the name of the Group that is written to the file. If this argument is omitted, users will be ' - 'required to enter a ``name`` field when creating instances of this data type in the API. Another ' - 'option is to specify ``default_name``, in which case this name will be used as the name of the Group ' - 'if no other name is provided.', - 'default': None, - }, - {'name': 'default_name', 'type': str, 'doc': 'The default name of this group', 'default': None}, - {'name': 'groups', 'type': list, 'doc': 'the subgroups in this group', 'default': list()}, - {'name': 'datasets', 'type': list, 'doc': 'the datasets in this group', 'default': list()}, - {'name': 'attributes', 'type': list, 'doc': 'the attributes on this group', 'default': list()}, - {'name': 'links', 'type': list, 'doc': 'the links in this group', 'default': list()}, - {'name': 'linkable', 'type': bool, 'doc': 'whether or not this group can be linked', 'default': True}, - { - 'name': 'quantity', - 'type': (str, int), - 'doc': "the allowable number of instance of this group in a certain location. See table of options " - "`here `_. Note that if you" - "specify ``name``, ``quantity`` cannot be ``'*'``, ``'+'``, or an integer greater that 1, because you " - "cannot have more than one group of the same name in the same parent group.", - 'default': 1, - }, - {'name': 'data_type_def', 'type': str, 'doc': 'the data type this specification represents', 'default': None}, - {'name': 'data_type_inc', 'type': str, - 'doc': 'the data type this specification data_type_inc', 'default': None}, -] - - -class GroupSpec(BaseStorageSpec): - ''' Specification for groups - ''' - - @docval(*_group_args) - def __init__(self, **kwargs): - doc, groups, datasets, links = popargs('doc', 'groups', 'datasets', 'links', kwargs) - self.__data_types = dict() # for GroupSpec/DatasetSpec data_type_def/inc - self.__target_types = dict() # for LinkSpec target_types - self.__groups = dict() - for group in groups: - self.set_group(group) - self.__datasets = dict() - for dataset in datasets: - self.set_dataset(dataset) - self.__links = dict() - for link in links: - self.set_link(link) - self.__new_data_types = set(self.__data_types.keys()) - self.__new_target_types = set(self.__target_types.keys()) - self.__new_datasets = set(self.__datasets.keys()) - self.__overridden_datasets = set() - self.__new_links = set(self.__links.keys()) - self.__overridden_links = set() - self.__new_groups = set(self.__groups.keys()) - self.__overridden_groups = set() - super().__init__(doc, **kwargs) - - def resolve_inc_spec(self, inc_spec: 'GroupSpec', namespace: 'SpecNamespace'): # noqa: C901 - """Add groups, datasets, links, and attributes from the inc_spec to this spec and track which ones are new and - overridden. - - Note that data_types and target_types are not added to this spec, but are used to determine if any datasets or - links need to be added to this spec. - - Parameters - ---------- - inc_spec : GroupSpec - The GroupSpec to inherit from - namespace : SpecNamespace - The namespace containing the specs - """ - if not isinstance(inc_spec, GroupSpec): # TODO: replace with Pydantic type checking - raise TypeError("Cannot resolve included spec: expected GroupSpec, got %s" % type(inc_spec)) - data_types = list() - target_types = list() - # resolve inherited datasets - for dataset in inc_spec.datasets: - if dataset.name is None: - data_types.append(dataset) - continue - self.__new_datasets.discard(dataset.name) - if dataset.name in self.__datasets: - # check compatibility between data_type_inc of the existing dataset spec and the included dataset spec - if ( - dataset.data_type_inc != self.__datasets[dataset.name].data_type_inc and - (dataset.data_type_inc is None or self.__datasets[dataset.name].data_type_inc is None or - dataset.data_type_inc not in namespace.get_hierarchy(self.__datasets[dataset.name].data_type_inc) - ) - ): - msg = ("Cannot resolve included dataset spec '%s' with data_type_inc '%s' because a dataset " - "spec with the same name already exists with data_type_inc '%s', and data type '%s' " - "is not a child type of data type '%s'." - % (dataset.name, dataset.data_type_inc, self.__datasets[dataset.name].data_type_inc, - self.__datasets[dataset.name].data_type_inc, dataset.data_type_inc)) - raise ValueError(msg) - - # if the included dataset spec was added earlier during resolution, don't add it again - # but resolve the spec using the included dataset spec - the included spec may contain - # properties not specified in the version of this spec added earlier during resolution - self.__datasets[dataset.name].resolve_inc_spec(dataset, namespace) - self.__overridden_datasets.add(dataset.name) - else: - self.set_dataset(dataset) - # resolve inherited groups - for group in inc_spec.groups: - if group.name is None: - data_types.append(group) - continue - self.__new_groups.discard(group.name) - if group.name in self.__groups: - # check compatibility between data_type_inc of the existing group spec and the included group spec - if ( - group.data_type_inc != self.__groups[group.name].data_type_inc and - (group.data_type_inc is None or self.__groups[group.name].data_type_inc is None or - group.data_type_inc not in namespace.get_hierarchy(self.__groups[group.name].data_type_inc) - ) - ): - msg = ("Cannot resolve included group spec '%s' with data_type_inc '%s' because a group " - "spec with the same name already exists with data_type_inc '%s', and data type '%s' " - "is not a child type of data type '%s'." - % (group.name, group.data_type_inc, self.__groups[group.name].data_type_inc, - self.__groups[group.name].data_type_inc, group.data_type_inc)) - raise ValueError(msg) - - # if the included group spec was added earlier during resolution, don't add it again - # but resolve the spec using the included group spec - the included spec may contain - # properties not specified in the version of this spec added earlier during resolution - self.__groups[group.name].resolve_inc_spec(group, namespace) - self.__overridden_groups.add(group.name) - else: - self.set_group(group) - # resolve inherited links - for link in inc_spec.links: - if link.name is None: - target_types.append(link) - continue - self.__new_links.discard(link.name) - if link.name in self.__links: - # TODO: check compatibility between target_type of the existing link spec and the included link spec - self.__overridden_links.add(link.name) - else: - self.set_link(link) - # resolve inherited data_types - for dt_spec in data_types: - dt = dt_spec.data_type_def - if dt is None: - dt = dt_spec.data_type_inc - self.__new_data_types.discard(dt) - existing_dt_spec = self.get_data_type(dt) - if (existing_dt_spec is None or - ((isinstance(existing_dt_spec, list) or existing_dt_spec.name is not None) and - dt_spec.name is None)): - if isinstance(dt_spec, DatasetSpec): - self.set_dataset(dt_spec) - else: - self.set_group(dt_spec) - # resolve inherited target_types - for link_spec in target_types: - dt = link_spec.target_type - self.__new_target_types.discard(dt) - existing_dt_spec = self.get_target_type(dt) - if (existing_dt_spec is None or - (isinstance(existing_dt_spec, list) or existing_dt_spec.name is not None) and - link_spec.name is None): - self.set_link(link_spec) - super().resolve_inc_spec(inc_spec, namespace) - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the dataset'}, - raises="ValueError, if 'name' is not part of this spec") - def is_inherited_dataset(self, **kwargs): - '''Return true if a dataset with the given name was inherited''' - name = getargs('name', kwargs) - if name not in self.__datasets: - raise ValueError("Dataset '%s' not found in spec" % name) - return name not in self.__new_datasets - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the dataset'}, - raises="ValueError, if 'name' is not part of this spec") - def is_overridden_dataset(self, **kwargs): - '''Return true if a dataset with the given name overrides a specification from the parent type''' - name = getargs('name', kwargs) - if name not in self.__datasets: - raise ValueError("Dataset '%s' not found in spec" % name) - return name in self.__overridden_datasets - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the group'}, - raises="ValueError, if 'name' is not part of this spec") - def is_inherited_group(self, **kwargs): - '''Return true if a group with the given name was inherited''' - name = getargs('name', kwargs) - if name not in self.__groups: - raise ValueError("Group '%s' not found in spec" % name) - return name not in self.__new_groups - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the group'}, - raises="ValueError, if 'name' is not part of this spec") - def is_overridden_group(self, **kwargs): - '''Return true if a group with the given name overrides a specification from the parent type''' - name = getargs('name', kwargs) - if name not in self.__groups: - raise ValueError("Group '%s' not found in spec" % name) - return name in self.__overridden_groups - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the link'}, - raises="ValueError, if 'name' is not part of this spec") - def is_inherited_link(self, **kwargs): - '''Return true if a link with the given name was inherited''' - name = getargs('name', kwargs) - if name not in self.__links: - raise ValueError("Link '%s' not found in spec" % name) - return name not in self.__new_links - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the link'}, - raises="ValueError, if 'name' is not part of this spec") - def is_overridden_link(self, **kwargs): - '''Return true if a link with the given name overrides a specification from the parent type''' - name = getargs('name', kwargs) - if name not in self.__links: - raise ValueError("Link '%s' not found in spec" % name) - return name in self.__overridden_links - - @docval({'name': 'spec', 'type': Spec, 'doc': 'the specification to check'}) - def is_inherited_spec(self, **kwargs): - ''' Returns 'True' if specification was inherited from a parent type ''' - spec = getargs('spec', kwargs) - spec_name = spec.name - if spec_name is None and hasattr(spec, 'data_type_def'): - spec_name = spec.data_type_def - if spec_name is None: # NOTE: this will return the target type for LinkSpecs - spec_name = spec.data_type_inc - if spec_name is None: # pragma: no cover - # this should not be possible - raise ValueError('received Spec with wildcard name but no data_type_inc or data_type_def') - # if the spec has a name, it will be found in __links/__groups/__datasets before __data_types/__target_types - if spec_name in self.__links: - return self.is_inherited_link(spec_name) - elif spec_name in self.__groups: - return self.is_inherited_group(spec_name) - elif spec_name in self.__datasets: - return self.is_inherited_dataset(spec_name) - elif spec_name in self.__data_types: - # NOTE: the same data type can be both an unnamed data type and an unnamed target type - return self.is_inherited_type(spec_name) - elif spec_name in self.__target_types: - return self.is_inherited_target_type(spec_name) - elif super().is_inherited_spec(spec): # attribute spec - return True - else: - parent_name = spec.parent.name - if parent_name is None: - parent_name = spec.parent.data_type - if isinstance(spec.parent, DatasetSpec): - if (parent_name in self.__datasets and self.is_inherited_dataset(parent_name) and - self.__datasets[parent_name].get_attribute(spec_name) is not None): - return True - else: - if (parent_name in self.__groups and self.is_inherited_group(parent_name) and - self.__groups[parent_name].get_attribute(spec_name) is not None): - return True - return False - - @docval({'name': 'spec', 'type': Spec, 'doc': 'the specification to check'}) - def is_overridden_spec(self, **kwargs): - ''' Returns 'True' if specification overrides a specification from the parent type ''' - spec = getargs('spec', kwargs) - spec_name = spec.name - if spec_name is None: - if isinstance(spec, LinkSpec): # unnamed LinkSpec cannot be overridden - return False - if spec.is_many(): # this is a wildcard spec, so it cannot be overridden - return False - spec_name = spec.data_type_def - if spec_name is None: # NOTE: this will return the target type for LinkSpecs - spec_name = spec.data_type_inc - if spec_name is None: # pragma: no cover - # this should not happen - raise ValueError('received Spec with wildcard name but no data_type_inc or data_type_def') - # if the spec has a name, it will be found in __links/__groups/__datasets before __data_types/__target_types - if spec_name in self.__links: - return self.is_overridden_link(spec_name) - elif spec_name in self.__groups: - return self.is_overridden_group(spec_name) - elif spec_name in self.__datasets: - return self.is_overridden_dataset(spec_name) - elif spec_name in self.__data_types: - return self.is_overridden_type(spec_name) - elif super().is_overridden_spec(spec): # attribute spec - return True - else: - parent_name = spec.parent.name - if parent_name is None: - parent_name = spec.parent.data_type - if isinstance(spec.parent, DatasetSpec): - if (parent_name in self.__datasets and self.is_overridden_dataset(parent_name) and - self.__datasets[parent_name].is_overridden_spec(spec)): - return True - else: - if (parent_name in self.__groups and self.is_overridden_group(parent_name) and - self.__groups[parent_name].is_overridden_spec(spec)): - return True - return False - - @docval({'name': 'spec', 'type': (BaseStorageSpec, str), 'doc': 'the specification to check'}) - def is_inherited_type(self, **kwargs): - ''' Returns True if `spec` represents a data type that was inherited ''' - spec = getargs('spec', kwargs) - if isinstance(spec, BaseStorageSpec): - if spec.data_type_def is None: # why not also check data_type_inc? - raise ValueError('cannot check if something was inherited if it does not have a %s' % self.def_key()) - spec = spec.data_type_def - return spec not in self.__new_data_types - - @docval({'name': 'spec', 'type': (BaseStorageSpec, str), 'doc': 'the specification to check'}, - raises="ValueError, if 'name' is not part of this spec") - def is_overridden_type(self, **kwargs): - ''' Returns True if `spec` represents a data type that overrides a specification from a parent type ''' - return self.is_inherited_type(**kwargs) - - @docval({'name': 'spec', 'type': (LinkSpec, str), 'doc': 'the specification to check'}) - def is_inherited_target_type(self, **kwargs): - ''' Returns True if `spec` represents a target type that was inherited ''' - spec = getargs('spec', kwargs) - if isinstance(spec, LinkSpec): - spec = spec.target_type - return spec not in self.__new_target_types - - @docval({'name': 'spec', 'type': (LinkSpec, str), 'doc': 'the specification to check'}, - raises="ValueError, if 'name' is not part of this spec") - def is_overridden_target_type(self, **kwargs): - ''' Returns True if `spec` represents a target type that overrides a specification from a parent type ''' - return self.is_inherited_target_type(**kwargs) - - def __add_data_type_inc(self, spec): - # update the __data_types dict with the given groupspec/datasetspec so that: - # - if there is only one spec for a given data type, then it is stored in __data_types regardless of - # whether it is named - # - if there are multiple specs for a given data type and they are all named, then they are all stored in - # __data_types - # - if there are multiple specs for a given data type and only one is unnamed, then the unnamed spec is - # stored in __data_types - # it is not allowed to have multiple specs for a given data type and multiple are unnamed - dt = None - if hasattr(spec, 'data_type_def') and spec.data_type_def is not None: - dt = spec.data_type_def - elif hasattr(spec, 'data_type_inc') and spec.data_type_inc is not None: - dt = spec.data_type_inc - if not dt: # pragma: no cover - # this should not be possible - raise TypeError("spec does not have '%s' or '%s' defined" % (self.def_key(), self.inc_key())) - if dt in self.__data_types: - curr = self.__data_types[dt] - if curr is spec: # happens only if the same spec is added twice - return - if spec.name is None: - if isinstance(curr, list): - # replace the list of named specs associated with the data_type with this unnamed spec - # the named specs can be retrieved by name - self.__data_types[dt] = spec - else: - if curr.name is None: - # neither the spec already associated with the data_type nor the given spec have a name - msg = "Cannot have multiple groups/datasets with the same data type without specifying name" - raise ValueError(msg) - else: - # replace the named spec associated with the data_type with this unnamed spec - # the named spec can be retrieved by name - self.__data_types[dt] = spec - else: - if isinstance(curr, list): - # add this named spec to the list of named current specs associated with the data_type - self.__data_types[dt].append(spec) - else: - if curr.name is None: - # the spec associated with the data_type has no name and the given spec has a name - # leave the existing data type as is, since the new one can be retrieved by name - return - else: - # both the spec associated with the data_type and the given spec have a name - # store both specific instances of a data type - self.__data_types[dt] = [curr, spec] - else: - self.__data_types[dt] = spec - - def __add_target_type(self, spec): - # update the __target_types dict with the given linkspec so that: - # - if there is only one linkspec for a given target type, then it is stored in __target_types regardless of - # whether it is named - # - if there are multiple linkspecs for a given target type and they are all named, then they are all stored in - # __target_types - # - if there are multiple linkspecs for a given target type and only one is unnamed, then the unnamed spec is - # stored in __target_types - # it is not allowed to have multiple linkspecs for a given target type and multiple are unnamed - dt = spec.target_type - if dt in self.__target_types: - curr = self.__target_types[dt] - if curr is spec: # happens only if the same spec is added twice - return - if spec.name is None: - if isinstance(curr, list): - # replace the list of named specs associated with the target_type with this unnamed spec - # the named specs can be retrieved by name - self.__target_types[dt] = spec - else: - if curr.name is None: - # neither the spec already associated with the target_type nor the given spec have a name - msg = "Cannot have multiple links with the same target type without specifying name" - raise ValueError(msg) - else: - # replace the named spec associated with the target_type with this unnamed spec - # the named spec can be retrieved by name - self.__target_types[dt] = spec - else: - if isinstance(curr, list): - # add this named spec to the list of named current specs associated with the target_type - self.__target_types[dt].append(spec) - else: - if curr.name is None: - # the spec associated with the target_type has no name and the given spec has a name - # leave the existing data type as is, since the new one can be retrieved by name - return - else: - # both the spec associated with the target_type and the given spec have a name - # store both specific instances of a data type - self.__target_types[dt] = [curr, spec] - else: - self.__target_types[dt] = spec - - @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to retrieve'}) - def get_data_type(self, **kwargs): - ''' Get a specification by "data_type" - - NOTE: If there is only one spec for a given data type, then it is returned. - If there are multiple specs for a given data type and they are all named, then they are returned in a list. - If there are multiple specs for a given data type and only one is unnamed, then the unnamed spec is returned. - The other named specs can be returned using get_group or get_dataset. - - NOTE: this method looks for an exact match of the data type and does not consider the type hierarchy. - ''' - ndt = getargs('data_type', kwargs) - return self.__data_types.get(ndt, None) - - @docval({'name': 'target_type', 'type': str, 'doc': 'the target_type to retrieve'}) - def get_target_type(self, **kwargs): - ''' Get a specification by "target_type" - - NOTE: If there is only one spec for a given target type, then it is returned. - If there are multiple specs for a given target type and they are all named, then they are returned in a list. - If there are multiple specs for a given target type and only one is unnamed, then the unnamed spec is returned. - The other named specs can be returned using get_link. - - NOTE: this method looks for an exact match of the target type and does not consider the type hierarchy. - ''' - ndt = getargs('target_type', kwargs) - return self.__target_types.get(ndt, None) - - @property - def groups(self): - ''' The groups specified in this GroupSpec ''' - return tuple(self.get('groups', tuple())) - - @property - def datasets(self): - ''' The datasets specified in this GroupSpec ''' - return tuple(self.get('datasets', tuple())) - - @property - def links(self): - ''' The links specified in this GroupSpec ''' - return tuple(self.get('links', tuple())) - - @docval(*_group_args) - def add_group(self, **kwargs): - ''' Add a new specification for a subgroup to this group specification ''' - spec = self.__class__(**kwargs) - self.set_group(spec) - return spec - - @docval({'name': 'spec', 'type': ('GroupSpec'), 'doc': 'the specification for the subgroup'}) - def set_group(self, **kwargs): - ''' Add the given specification for a subgroup to this group specification ''' - spec = getargs('spec', kwargs) - if spec.parent is not None: - spec = self.build_spec(spec) - if spec.name is None: - if spec.data_type_inc is not None or spec.data_type_def is not None: - self.__add_data_type_inc(spec) - else: # pragma: no cover - # this should not be possible - raise TypeError("must specify 'name' or 'data_type_inc' in Group spec") - else: - # NOTE named specs can be present in both __datasets and __data_types - if spec.data_type_inc is not None or spec.data_type_def is not None: - self.__add_data_type_inc(spec) - self.__groups[spec.name] = spec - self.setdefault('groups', list()).append(spec) - spec.parent = self - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the group to the Spec for'}) - def get_group(self, **kwargs): - ''' Get a specification for a subgroup to this group specification ''' - name = getargs('name', kwargs) - return self.__groups.get(name, self.__links.get(name)) - - @docval(*_dataset_args) - def add_dataset(self, **kwargs): - ''' Add a new specification for a dataset to this group specification ''' - spec = self.dataset_spec_cls()(**kwargs) - self.set_dataset(spec) - return spec - - @docval({'name': 'spec', 'type': 'hdmf.spec.spec.DatasetSpec', 'doc': 'the specification for the dataset'}) - def set_dataset(self, **kwargs): - ''' Add the given specification for a dataset to this group specification ''' - spec = getargs('spec', kwargs) - if spec.parent is not None: - spec = self.dataset_spec_cls().build_spec(spec) - if spec.name is None: - if spec.data_type_inc is not None or spec.data_type_def is not None: - self.__add_data_type_inc(spec) - else: # pragma: no cover - # this should not be possible - raise TypeError("must specify 'name' or 'data_type_inc' in Dataset spec") - else: - # NOTE named specs can be present in both __datasets and __data_types - if spec.data_type_inc is not None or spec.data_type_def is not None: - self.__add_data_type_inc(spec) - self.__datasets[spec.name] = spec - self.setdefault('datasets', list()).append(spec) - spec.parent = self - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the dataset to the Spec for'}) - def get_dataset(self, **kwargs): - ''' Get a specification for a dataset to this group specification ''' - name = getargs('name', kwargs) - return self.__datasets.get(name, self.__links.get(name)) - - @docval(*_link_args) - def add_link(self, **kwargs): - ''' Add a new specification for a link to this group specification ''' - spec = self.link_spec_cls()(**kwargs) - self.set_link(spec) - return spec - - @docval({'name': 'spec', 'type': 'hdmf.spec.spec.LinkSpec', 'doc': 'the specification for the object to link to'}) - def set_link(self, **kwargs): - ''' Add a given specification for a link to this group specification ''' - spec = getargs('spec', kwargs) - if spec.parent is not None: - spec = self.link_spec_cls().build_spec(spec) - # NOTE named specs can be present in both __links and __target_types - self.__add_target_type(spec) - if spec.name is not None: - self.__links[spec.name] = spec - self.setdefault('links', list()).append(spec) - spec.parent = self - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the link to the Spec for'}) - def get_link(self, **kwargs): - ''' Get a specification for a link to this group specification ''' - name = getargs('name', kwargs) - return self.__links.get(name) - - @classmethod - def dataset_spec_cls(cls): - ''' The class to use when constructing DatasetSpec objects - - Override this if extending to use a class other than DatasetSpec to build - dataset specifications - ''' - return DatasetSpec - - @classmethod - def link_spec_cls(cls): - ''' The class to use when constructing LinkSpec objects - - Override this if extending to use a class other than LinkSpec to build - link specifications - ''' - return LinkSpec - - @classmethod - def build_const_args(cls, spec_dict): - ''' Build constructor arguments for this Spec class from a dictionary ''' - ret = super().build_const_args(spec_dict) - if 'datasets' in ret: - ret['datasets'] = list(map(cls.dataset_spec_cls().build_spec, ret['datasets'])) - if 'groups' in ret: - ret['groups'] = list(map(cls.build_spec, ret['groups'])) - if 'links' in ret: - ret['links'] = list(map(cls.link_spec_cls().build_spec, ret['links'])) - return ret diff --git a/src/hdmf/spec/spec2.py b/src/hdmf/spec/spec2.py new file mode 100644 index 000000000..c6ee6b13c --- /dev/null +++ b/src/hdmf/spec/spec2.py @@ -0,0 +1,1007 @@ +from abc import ABC, abstractmethod +from enum import Enum +from itertools import chain +from typing import Any, ClassVar, Literal, Optional, Self, Union + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator, PositiveInt, validate_call + + +# TODO: breaking change. specs are no longer dicts, no longer mutable, and no longer accessible via getitem +# no longer supports positional args. dims and shape are always converted to tuples if provided as lists +# TODO: error messages have changed +# DtypeHelper.check_dtype no longer returns the dtype, it just raises an error if invalid +# TODO: removed support for DatasetSpec.default_value +# TODO: removed support for BaseStorageSpec.linkable + +class DtypeHelper: + # Dict where the keys are the primary data type and the values are list of strings with synonyms for the dtype + # make sure keys are consistent between hdmf.spec.spec.DtypeHelper.primary_dtype_synonyms, + # hdmf.build.objectmapper.ObjectMapper.__dtypes, hdmf.build.manager.TypeMap._spec_dtype_map, + # hdmf.validate.validator.__allowable, and backend dtype maps + # see https://hdmf-schema-language.readthedocs.io/en/latest/description.html#dtype + primary_dtype_synonyms = { + 'float': ["float", "float32"], + 'double': ["double", "float64"], + 'short': ["int16", "short"], + 'int': ["int32", "int"], + 'long': ["int64", "long"], + 'utf': ["text", "utf", "utf8", "utf-8"], + 'ascii': ["ascii", "bytes"], + 'bool': ["bool"], + 'int8': ["int8"], + 'uint8': ["uint8"], + 'uint16': ["uint16"], + 'uint32': ["uint32", "uint"], + 'uint64': ["uint64"], + 'object': ['object'], + 'numeric': ['numeric'], + 'isodatetime': ["isodatetime", "datetime", "date"] + } + + # List of recommended primary dtype strings. These are the keys of primary_dtype_string_synonyms + recommended_primary_dtypes = list(primary_dtype_synonyms.keys()) + + # List of valid primary data type strings + valid_primary_dtypes = set(list(primary_dtype_synonyms.keys()) + + [vi for v in primary_dtype_synonyms.values() for vi in v]) + + # TODO: evaluate whether this is necessary + @staticmethod + def simplify_cpd_type(cpd_type: list['DtypeSpec']) -> list[str]: + ''' + Transform a list of DtypeSpecs into a list of strings. + Use for simple representation of compound type and validation. + + For example, a compound type with fields of dtype 'float' and 'int' + will be transformed to ['float', 'int'], and a compound type with fields + of dtype 'float' and RefSpec(target_type='MyType', reftype='object') + will be transformed to ['float', 'object']. + + Parameters + ---------- + cpd_type : list[DtypeSpec] + The compound type to simplify + + Returns + ------- + list[str] + A list of strings representing the simplified compound type. + + ''' + ret = list() + for exp in cpd_type: + exp_key = exp.dtype + if isinstance(exp_key, RefSpec): + exp_key = exp_key.reftype + ret.append(exp_key) + return ret + + @staticmethod + def check_dtype(dtype: Optional[Union[str, list['DtypeSpec'], 'RefSpec']]) -> None: + """Check that the dtype string is a reference or a valid primary dtype.""" + if not isinstance(dtype, RefSpec) and dtype not in DtypeHelper.valid_primary_dtypes: + raise ValueError("dtype '%s' is not a valid primary data type. Allowed dtypes: %s" + % (dtype, str(DtypeHelper.valid_primary_dtypes))) + + # all keys and values should be keys in primary_dtype_synonyms + additional_allowed = { + 'float': ['double'], + 'int8': ['short', 'int', 'long'], + 'short': ['int', 'long'], + 'int': ['long'], + 'uint8': ['uint16', 'uint32', 'uint64'], + 'uint16': ['uint32', 'uint64'], + 'uint32': ['uint64'], + 'utf': ['ascii'] + } + + # if the spec dtype is a key in __allowable, then all types in __allowable[key] are valid + allowable = dict() + for dt, dt_syn in primary_dtype_synonyms.items(): + allow = dt_syn.copy() + if dt in additional_allowed: + for addl in additional_allowed[dt]: + allow.extend(primary_dtype_synonyms[addl]) + for syn in dt_syn: + allowable[syn] = allow + allowable['numeric'].extend(set(chain.from_iterable(v for k, v in allowable.items() if 'int' in k or 'float' in k))) + + @staticmethod + def is_allowed_dtype(new: str, orig: str): + if orig not in DtypeHelper.allowable: + raise ValueError(f"Invalid dtype '{orig}'.") + return new in DtypeHelper.allowable[orig] + + +class Spec(BaseModel, ABC): + """ + A base specification class. + """ + model_config = ConfigDict(extra='forbid', validate_assignment=True, validate_default=True) + + doc: str + """Documentation on what this specification is specifying.""" + + name: Optional[str] = None + """The name of the object being specified.""" + + _parent: Optional["Spec"] = None + """The parent specification of this specification.""" + + @property + def parent(self) -> Optional["Spec"]: + return self._parent + + @parent.setter + def parent(self, value: "Spec") -> None: + if self._parent is not None and self._parent != value: + raise ValueError('Parent cannot be changed after being set.') + self._parent = value + + # TODO: make hashable + + @property + def path(self) -> str: + """The full path of this specification in the hierarchy of specifications.""" + stack = list() + tmp = self + while tmp is not None: + name = tmp.name or tmp.data_type_def or tmp.data_type_inc + # name = tmp.name + # if name is None: + # name = tmp.data_type_def + # if name is None: + # name = tmp.data_type_inc + stack.append(name) + tmp = tmp.parent + return "/".join(reversed(stack)) + +Spec.parent = Spec.parent.setter(validate_call(config=dict(arbitrary_types_allowed=True))(Spec.parent.fset)) + + +class RefSpec(BaseModel): + """ + A specification that references another specification. + """ + model_config = ConfigDict(extra='forbid', validate_assignment=True, validate_default=True) + + target_type: str + """The type of object that this specification references (e.g., Group, Dataset, etc.).""" + + reftype: Literal["object"] + """The type of reference. Only "object" is supported.""" + + +class AttributeSpec(Spec): + """ + A specification for an attribute. + """ + + required: Optional[bool] = True + """Whether or not this specification is required. Default is True.""" + + dtype: Union[str, RefSpec] + """The data type of the attribute.""" + # NOTE: compound data types are not supported in attributes + + shape: Optional[tuple] = None + """The shape of the attribute.""" + + dims: Optional[tuple] = None + """The dimensions of the attribute.""" + + value: Optional[Any] = None + """A constant value for this attribute.""" + + default_value: Optional[Any] = None + """A default value for this attribute.""" + + @model_validator(mode="before") + def set_dims_based_on_shape(cls, data: dict) -> dict: + """If 'shape' is provided but 'dims' is not, set 'dims' to default dimension names.""" + if 'shape' in data and data['shape'] is not None and ('dims' not in data or data['dims'] is None): + data['dims'] = tuple(f"dim_{i}" for i in range(len(data['shape']))) + return data + + @model_validator(mode="before") + def set_shape_based_on_dims(cls, data: dict) -> dict: + """If 'dims' is provided but 'shape' is not, set 'shape' to (None,)*len(dims).""" + if 'dims' in data and data['dims'] is not None and ('shape' not in data or data['shape'] is None): + data['shape'] = tuple(None for _ in range(len(data['dims']))) + return data + + @model_validator(mode="before") + def set_optional_if_default_value(cls, data: dict) -> dict: + """If 'default_value' is provided, set 'required' to False.""" + if 'default_value' in data and data['default_value'] is not None: + data['required'] = False # TODO handle not setting this so that it is not printed in model_dump + return data + + @model_validator(mode="after") + def check_dtype_valid(self) -> Self: + DtypeHelper.check_dtype(self.dtype) + return self + + @model_validator(mode="after") + def check_not_both_value_and_default_value(self) -> Self: + if (self.value is not None and self.default_value is not None): + raise ValueError("Cannot specify both 'value' and 'default_value'.") + return self + + @model_validator(mode="after") + def check_not_both_value_and_optional(self) -> Self: + if self.value is not None and self.required is False: + raise ValueError("Cannot specify 'value' and 'required=False' at the same time.") + return self + + @model_validator(mode="after") + def check_dims_and_shape_same_length(self) -> Self: + if self.dims is not None and self.shape is not None: + if len(self.dims) != len(self.shape): + raise ValueError("'dims' and 'shape' must have the same length.") + return self + + @classmethod + def build_spec(cls, spec_dict: dict[str, Any]) -> 'AttributeSpec': + ''' Build a specification from a dictionary + + Parameters + ---------- + spec_dict : dict[str, Any] + The dictionary to build the specification from + + Returns + ------- + AttributeSpec + The constructed AttributeSpec object + ''' + input_dict = spec_dict.copy() + if isinstance(input_dict['dtype'], dict): + input_dict['dtype'] = RefSpec(**input_dict['dtype']) + return cls(**input_dict) + + +class QuantityEnum(str, Enum): + """ + An enum for quantity values. + + The ZERO_OR_ONE value can be represented by either '?' or 'zero_or_one'. + The ZERO_OR_MANY value can be represented by either '*' or 'zero_or_many'. + The ONE_OR_MANY value can be represented by either '+' or 'one_or_many'. + """ + ZERO_OR_ONE = '?' + ZERO_OR_MANY = '*' + ONE_OR_MANY = '+' + + @classmethod + def _missing_(cls, value): + """Allow string aliases to map to enum values.""" + if value == 'zero_or_one': + return cls.ZERO_OR_ONE + elif value == 'zero_or_many': + return cls.ZERO_OR_MANY + elif value == 'one_or_many': + return cls.ONE_OR_MANY + return None + + +class BaseStorageSpec(Spec, ABC): + """ + A base specification for groups and datasets. + """ + + __inc_key: ClassVar[str] = 'data_type_inc' + __def_key: ClassVar[str] = 'data_type_def' + __type_key: ClassVar[str] = 'data_type' + __id_key: ClassVar[str] = 'object_id' + + data_type_def: Optional[str] = None + """The data type that this specification defines.""" + + data_type_inc: Optional[str] = None + """The data type that this specification extends.""" + + default_name: Optional[str] = None + """The default name of this specification.""" + + quantity: Union[PositiveInt, QuantityEnum] = 1 + """The quantity of this specification. Can be a positive integer or one of '?' (zero or one), '*' (zero or more), + '+' (one or more).""" + + linkable: bool = Field(default=False, exclude=True) + """DEPRECATED and ignored. Maintained for backwards compatibility. Whether this specification can be linked to.""" + + attributes: list[AttributeSpec] = list() + """A list of attribute specifications for this specification.""" + + _not_inherited_attributes: dict[str, AttributeSpec] = dict() + """A dictionary of attribute specifications that are defined on this specification and not inherited from + an included data type (data_type_inc).""" + # TODO single or double underscore? + # TODO allow these to be modified + + _overridden_attributes: dict[str, AttributeSpec] = dict() + """A dictionary of attribute specifications that override attributes from an included data type + (data_type_inc).""" + + __inc_spec_resolved: bool = False + """Whether or not this specification has been fully resolved (i.e., fields from an included data type + (data_type_inc) have been merged into this specification).""" + + __resolved: bool = False + """Whether or not this specification has been fully resolved (i.e., fields from an included data type + (data_type_inc) and any fields from subspecs with an included data type (data_type_inc) have been merged + into this specification).""" + + def model_post_init(self, context: Any) -> None: + """ Set parent on attribute specs after initialization """ + for attr in self.attributes: + attr.parent = self + + @field_validator("name", mode="after") + def check_valid_name(cls, v: Optional[str]) -> Optional[str]: + if v is not None and '/' in v: + raise ValueError("Invalid character '/' in 'name'.") + return v + + @field_validator("default_name", mode="after") + def check_valid_default_name(cls, v: Optional[str]) -> Optional[str]: + if v is not None and '/' in v: + raise ValueError("Invalid character '/' in 'default_name'.") + return v + + @model_validator(mode="after") + def check_name_or_data_type_def_or_data_type_inc(self) -> Self: + if self.name is None and self.data_type_def is None and self.data_type_inc is None: + raise ValueError("At least one of 'name', 'data_type_def', or 'data_type_inc' must be specified.") + return self + + @model_validator(mode="after") + def check_not_both_name_and_default_name(self) -> Self: + if self.name is not None and self.default_name is not None: + raise ValueError("Cannot specify both 'name' and 'default_name'.") + return self + + @model_validator(mode="after") + def check_not_name_and_many_quantity(self) -> Self: + if self.name is not None: + if isinstance(self.quantity, int): + if self.quantity != 1: + raise ValueError("Cannot specify 'name' with a quantity other than 1.") + elif self.is_many(): + raise ValueError("Cannot specify 'name' on a spec that can exist multiple times.") + return self + + @model_validator(mode="after") + def check_data_type_def_not_equal_data_type_inc(self) -> Self: + if self.data_type_def is not None and self.data_type_inc is not None: + if self.data_type_def == self.data_type_inc: + raise ValueError( + f"Cannot specify the same value for 'data_type_def' and 'data_type_inc': {self.data_type_def}" + ) + return self + + def is_many(self) -> bool: + return self.quantity not in (1, QuantityEnum.ZERO_OR_ONE) + + @property + @validate_call + def inc_spec_resolved(self) -> bool: + return self.__inc_spec_resolved + + @property + @validate_call + def resolved(self) -> bool: + return self.__resolved + + @resolved.setter + @validate_call + def resolved(self, val: bool) -> None: + self.__resolved = val + + @property + def required(self) -> bool: + ''' Whether or not this specification is required. ''' + return self.quantity not in (QuantityEnum.ZERO_OR_ONE, QuantityEnum.ZERO_OR_MANY) + + # TODO add namespace: SpecNamespace hint + @abstractmethod + def resolve_inc_spec(self, inc_spec: 'BaseStorageSpec', namespace) -> None: + pass + + # TODO determine whether we need to track inherited and overridden attributes + + @classmethod + def id_key(cls) -> str: + ''' Get the key used to store data ID on an instance + + Override this method to use a different name for 'object_id' + ''' + return cls.__id_key + + @classmethod + def type_key(cls) -> str: + ''' Get the key used to store data type on an instance + + Override this method to use a different name for 'data_type'. HDMF supports combining schema + that uses 'data_type' and at most one different name for 'data_type'. + ''' + return cls.__type_key + + @classmethod + def inc_key(cls) -> str: + ''' Get the key used to define a data_type include. + + Override this method to use a different keyword for 'data_type_inc'. HDMF supports combining schema + that uses 'data_type_inc' and at most one different name for 'data_type_inc'. + ''' + return cls.__inc_key + + @classmethod + def def_key(cls) -> str: + ''' Get the key used to define a data_type definition. + + Override this method to use a different keyword for 'data_type_def' HDMF supports combining schema + that uses 'data_type_def' and at most one different name for 'data_type_def'. + ''' + return cls.__def_key + + @property + def data_type(self) -> Optional[str]: + ''' The data type of this specification ''' + return self.data_type_def or self.data_type_inc + + @validate_call + def set_attribute(self, spec: AttributeSpec) -> None: + ''' Add an attribute specification to this specification + + Parameters + ---------- + spec : AttributeSpec + The attribute specification to add + + Raises + ------ + ValueError + If an attribute with the same name already exists + ''' + # NOTE: this method would be better named add_attribute. however, there was previously a method named + # add_attribute that took the kwargs of AttributeSpec. this needed to be deprecated and removed to avoid + # confusion. + # TODO: copy logic from BaseStorageSpec.set_attribute + # TODO: You can no longer set attributes with the same name as an existing attribute (to replace it). + if spec.name in [attr.name for attr in self.attributes]: + raise ValueError(f"Attribute '{spec.name}' already exists in spec '{self.name or self.data_type}'.") + self.attributes.append(spec) + spec.parent = self + + @validate_call + def get_attribute(self, name: str) -> Optional[AttributeSpec]: + ''' Get an attribute specification by name + + Parameters + ---------- + name : str + The name of the attribute specification to get + + Returns + ------- + Optional[AttributeSpec] + The attribute specification with the given name, or None if it does not exist + ''' + return self.attributes.get(name, None) + + @classmethod + def build_spec(cls, spec_dict: dict[str, Any]) -> 'BaseStorageSpec': + ''' Build a specification from a dictionary + + Parameters + ---------- + spec_dict : dict[str, Any] + The dictionary to build the specification from + + Returns + ------- + BaseStorageSpec + The constructed BaseStorageSpec object + ''' + input_dict = spec_dict.copy() + if 'attributes' in input_dict: + input_dict['attributes'] = [AttributeSpec.build_spec(sub_spec) for sub_spec in input_dict['attributes']] + return cls(**input_dict) + +BaseStorageSpec.resolve_inc_spec = validate_call(config=dict(arbitrary_types_allowed=True))( + BaseStorageSpec.resolve_inc_spec +) +BaseStorageSpec.build_spec = validate_call(config=dict(arbitrary_types_allowed=True))(BaseStorageSpec.build_spec) + +class DtypeSpec(BaseModel): + """ + A specification for a field of a compound data type. + """ + model_config = ConfigDict(extra='forbid', validate_assignment=True, validate_default=True) + + name: str + """The name of this field in the compound data type.""" + + doc: str + """Documentation on what this field is specifying.""" + + dtype: Union[str, RefSpec] + """The data type of this field.""" + + # TODO: breaking change assertValidDtype and is_ref are removed + @model_validator(mode="after") + def check_dtype_valid(self) -> Self: + DtypeHelper.check_dtype(self.dtype) + return self + + @classmethod + def build_spec(cls, spec_dict: dict[str, Any]) -> 'DtypeSpec': + ''' Build a specification from a dictionary + + Parameters + ---------- + spec_dict : dict[str, Any] + The dictionary to build the specification from + + Returns + ------- + DtypeSpec + The constructed DtypeSpec object + ''' + input_dict = spec_dict.copy() + # TODO why this transformation? + # NOTE: Nesting DtypeSpecs is not supported + # if isinstance(ret['dtype'], list): + # input_dict['dtype'] = list(map(cls.build_const_args, input_dict['dtype'])) + if isinstance(input_dict['dtype'], dict): + input_dict['dtype'] = RefSpec(**input_dict['dtype']) + return cls(**input_dict) + + +class DatasetSpec(BaseStorageSpec): + """ + A specification for a dataset. + """ + + dtype: Optional[Union[str, list[DtypeSpec], RefSpec]] = None + """The data type of the dataset. Use a list of DtypeSpecs to specify a compound data type. + Use None for untyped datasets.""" + # Unlike AttributeSpec, compound data types are supported in DatasetSpec and dtype can be None for untyped datasets + + shape: Optional[tuple] = None + """The shape of the dataset.""" + # TODO: specify the type of the tuple elements (int or None) + + dims: Optional[tuple] = None + """The dimensions of the dataset.""" + + value: Optional[Any] = None + """A constant value for this dataset.""" + + default_value: Optional[Any] = None + """DEPRECATED and ignored. Maintained for backwards compatibility. A default value for this dataset.""" + + @model_validator(mode="before") + def set_dims_based_on_shape(cls, data: dict) -> dict: + """If 'shape' is provided but 'dims' is not, set 'dims' to default dimension names.""" + if 'shape' in data and data['shape'] is not None and ('dims' not in data or data['dims'] is None): + data['dims'] = tuple(f"dim_{i}" for i in range(len(data['shape']))) + return data + + @model_validator(mode="before") + def set_shape_based_on_dims(cls, data: dict) -> dict: + """If 'dims' is provided but 'shape' is not, set 'shape' to (None,)*len(dims).""" + if 'dims' in data and data['dims'] is not None and ('shape' not in data or data['shape'] is None): + data['shape'] = tuple(None for _ in range(len(data['dims']))) + return data + + @model_validator(mode="after") + def check_dtype_valid(self) -> Self: + if self.dtype is not None: + if isinstance(self.dtype, list): + for dt in self.dtype: + DtypeHelper.check_dtype(dt.dtype) + else: + DtypeHelper.check_dtype(self.dtype) + return self + + @model_validator(mode="after") + def check_not_both_value_and_optional(self) -> Self: + if self.value is not None and self.required is False: + raise ValueError("Cannot specify 'value' and 'required=False' at the same time.") + return self + + @model_validator(mode="after") + def check_dims_and_shape_same_length(self) -> Self: + if self.dims is not None and self.shape is not None: + if len(self.dims) != len(self.shape): + raise ValueError("'dims' and 'shape' must have the same length.") + return self + + # TODO add namespace: SpecNamespace hint + def resolve_inc_spec(self, inc_spec: 'DatasetSpec', namespace) -> None: + pass + + @classmethod + def dtype_spec_cls(cls) -> type[DtypeSpec]: + ''' The class to use when constructing DtypeSpec objects + + Override this if extending to use a class other than DtypeSpec to build + dataset specifications + ''' + return DtypeSpec + + @classmethod + def build_spec(cls, spec_dict: dict[str, Any]) -> 'DatasetSpec': + ''' Build a specification from a dictionary + + Parameters + ---------- + spec_dict : dict[str, Any] + The dictionary to build the specification from + + Returns + ------- + DatasetSpec + The constructed DatasetSpec object + ''' + input_dict = spec_dict.copy() + if 'attributes' in input_dict: + input_dict['attributes'] = [AttributeSpec.build_spec(sub_spec) for sub_spec in input_dict['attributes']] + if 'dtype' in input_dict: + if isinstance(input_dict['dtype'], list): # compound data type + input_dict['dtype'] = list(map(cls.dtype_spec_cls().build_spec, input_dict['dtype'])) + elif isinstance(input_dict['dtype'], dict): # reference to another spec + input_dict['dtype'] = RefSpec(**input_dict['dtype']) + return cls(**input_dict) + +DatasetSpec.resolve_inc_spec = validate_call(config=dict(arbitrary_types_allowed=True))(DatasetSpec.resolve_inc_spec) + + +class LinkSpec(Spec): + """ + A specification for a link. + """ + + target_type: str + """The target type GroupSpec or DatasetSpec.""" + + quantity: Union[PositiveInt, QuantityEnum] = 1 + """The quantity of this specification. Can be a positive integer or one of '?' (zero or one), '*' (zero or more), + '+' (one or more).""" + + name: Optional[str] = None + """The name of this link.""" + + @model_validator(mode="after") + def check_not_both_name_and_many_quantity(self) -> Self: + if self.name is not None: + if isinstance(self.quantity, int): + if self.quantity != 1: + raise ValueError("Cannot specify 'name' with a quantity other than 1.") + elif self.is_many(): + raise ValueError("Cannot specify 'name' on a spec that can exist multiple times.") + return self + + def is_many(self) -> bool: + return self.quantity not in (1, QuantityEnum.ZERO_OR_ONE) + + # TODO removed data_type_inc property + + # TODO can no longer set target_type to a BaseStorageSpec + + @property + def required(self) -> bool: + ''' Whether or not this specification is required. ''' + return self.quantity not in (QuantityEnum.ZERO_OR_ONE, QuantityEnum.ZERO_OR_MANY) + + +class GroupSpec(BaseStorageSpec): + """ + A specification for a group. + """ + + # NOTE: Some groups, datasets, and links do not have names + + groups: list['GroupSpec'] = list() + """A list of subgroup specifications for this group.""" + + datasets: list[DatasetSpec] = list() + """A list of dataset specifications for this group.""" + + links: list[LinkSpec] = list() + """A list of link specifications for this group.""" + + def model_post_init(self, context: Any) -> None: + """ Set parent on subspecs after initialization """ + for subspec in self.groups + self.datasets + self.links: + subspec.parent = self + super().model_post_init(context) + + @model_validator(mode="after") + def check_no_name_conflicts_groups_datasets_links(self) -> Self: + # Check for name conflicts between groups, datasets, and links. + # Conflicts with attribute names are not checked because attributes are stored separately. + all_names = set() + for spec in self.groups + self.datasets + self.links: + if spec.name is not None: + if spec.name in all_names: + msg = ( + f"Name conflict: '{spec.name}' is used in multiple places in group " + f"'{self.name or self.data_type}'." + ) + raise ValueError(msg) + all_names.add(spec.name) + return self + + @model_validator(mode="after") + def check_no_duplicate_data_type(self) -> Self: + # Check for duplicate data_type in groups and datasets. + # This is to prevent ambiguity when resolving data types. + all_data_types = set() + for spec in self.groups + self.datasets: + if spec.data_type is not None and spec.name is None: + if spec.data_type in all_data_types: + msg = ( + f"Duplicate data_type: '{spec.data_type}' is used in multiple unnamed subspecs in group " + f"'{self.name or self.data_type}'." + ) + raise ValueError(msg) + all_data_types.add(spec.data_type) + return self + + @model_validator(mode="after") + def check_no_duplicate_target_type(self) -> Self: + # Check for duplicate target_type in links. + # This is to prevent ambiguity when resolving target types. + all_target_types = set() + for spec in self.links: + if spec.name is None: + if spec.target_type in all_target_types: + msg = ( + f"Duplicate target_type: '{spec.target_type}' is used in multiple unnamed links in group " + f"'{self.name or self.data_type}'." + ) + raise ValueError(msg) + all_target_types.add(spec.target_type) + return self + + def set_group(self, spec: 'GroupSpec') -> None: + ''' Add a subgroup specification to this group spec + + Parameters + ---------- + spec : GroupSpec + The subgroup specification to add + + Raises + ------ + ValueError + If a subgroup with the same name already exists + ''' + if spec.name is not None and self.get_group(spec.name) is not None: + raise ValueError(f"A subgroup with name '{spec.name}' already exists.") + if spec.name is None and self.get_data_type(spec.data_type) is not None: + raise ValueError(f"A spec with data_type '{spec.data_type}' already exists.") + if not self.groups: # explicitly set self.groups to update the "set" state of the field + self.groups = list() + self.groups.append(spec) + spec.parent = self + + @validate_call + def set_dataset(self, spec: DatasetSpec) -> None: + ''' Add a dataset specification to this group spec + + Parameters + ---------- + spec : DatasetSpec + The dataset specification to add + + Raises + ------ + ValueError + If a dataset with the same name already exists + ''' + if spec.name is not None and self.get_dataset(spec.name) is not None: + raise ValueError(f"A dataset with name '{spec.name}' already exists.") + if spec.name is None and self.get_data_type(spec.data_type) is not None: + raise ValueError(f"A spec with data_type '{spec.data_type}' already exists.") + if not self.datasets: # explicitly set self.datasets to update the "set" state of the field + self.datasets = list() + self.datasets.append(spec) + spec.parent = self + + @validate_call + def set_link(self, spec: LinkSpec) -> None: + ''' Add a link specification to this group spec + + Parameters + ---------- + spec : LinkSpec + The link specification to add + + Raises + ------ + ValueError + If a link with the same name already exists + ''' + if spec.name is not None and self.get_link(spec.name) is not None: + raise ValueError(f"A link with name '{spec.name}' already exists.") + if spec.name is None and self.get_target_type(spec.target_type) is not None: + raise ValueError(f"A spec with target_type '{spec.target_type}' already exists.") + if not self.links: # explicitly set self.links to update the "set" state of the field + self.links = list() + self.links.append(spec) + spec.parent = self + + def get_group(self, name: str) -> Optional['GroupSpec']: + ''' Get a subgroup specification by name + + Parameters + ---------- + name : str + The name of the subgroup specification to get + + Returns + ------- + Optional[GroupSpec] + The subgroup specification with the given name, or None if it does not exist + ''' + # Some groups may not have names, so we need to search the list. + for group in self.groups: + if group.name == name: + return group + return None + + @validate_call + def get_dataset(self, name: str) -> Optional[DatasetSpec]: + ''' Get a dataset specification by name + + Parameters + ---------- + name : str + The name of the dataset specification to get + + Returns + ------- + Optional[DatasetSpec] + The dataset specification with the given name, or None if it does not exist + ''' + # Some datasets may not have names, so we need to search the list. + for dataset in self.datasets: + if dataset.name == name: + return dataset + return None + + @validate_call + def get_link(self, name: str) -> Optional[LinkSpec]: + ''' Get a link specification by name + + Parameters + ---------- + name : str + The name of the link specification to get + + Returns + ------- + Optional[LinkSpec] + The link specification with the given name, or None if it does not exist + ''' + # Some links may not have names, so we need to search the list. + for link in self.links: + if link.name == name: + return link + return None + + def get_data_type(self, data_type: str = None) -> Optional[ + Union['GroupSpec', DatasetSpec, list[Union['GroupSpec', DatasetSpec]]] + ]: + ''' Get a GroupSpec or DatasetSpec by "data_type" + + The "data_type" for a spec is defined as the value of "data_type_def" if it exists, + otherwise the value of "data_type_inc". + + NOTE: If there is only one spec for a given data type, then it is returned. + If there are multiple specs for a given data type and they are all named, then they are returned in a list. + If there are multiple specs for a given data type and only one is unnamed, then the unnamed spec is returned. + The other named specs can be returned using get_group or get_dataset. + + NOTE: this method looks for an exact match of the data type and does not consider the type hierarchy. + ''' + # TODO: This is really confusing behavior. Consider always returning a list and letting the user handle it. + matching_specs = [spec for spec in (self.groups + self.datasets) if spec.data_type == data_type] + if len(matching_specs) == 1: + return matching_specs[0] + elif len(matching_specs) > 1: + unnamed_specs = [spec for spec in matching_specs if spec.name is None] + if len(unnamed_specs) == 1: + return unnamed_specs[0] + else: + return matching_specs + return None + + @validate_call + def get_target_type(self, target_type: str = None) -> Optional[Union[LinkSpec, list[LinkSpec]]]: + ''' Get a specification by "target_type" + + NOTE: If there is only one spec for a given target type, then it is returned. + If there are multiple specs for a given target type and they are all named, then they are returned in a list. + If there are multiple specs for a given target type and only one is unnamed, then the unnamed spec is returned. + The other named specs can be returned using get_link. + + NOTE: this method looks for an exact match of the target type and does not consider the type hierarchy. + ''' + # TODO: This is really confusing behavior. Consider always returning a list and letting the user handle it. + matching_specs = [spec for spec in self.links if spec.target_type == target_type] + if len(matching_specs) == 1: + return matching_specs[0] + elif len(matching_specs) > 1: + unnamed_specs = [spec for spec in matching_specs if spec.name is None] + if len(unnamed_specs) == 1: + return unnamed_specs[0] + else: + return matching_specs + return None + + # add namespace type hint + def resolve_inc_spec(self, inc_spec: 'GroupSpec', namespace) -> None: + pass + + # add is inherited / overridden tracking + + # TODO removed get_group, get_dataset, get_link methods + + @classmethod + def dataset_spec_cls(cls) -> type[DatasetSpec]: + ''' The class to use when constructing DatasetSpec objects + + Override this if extending to use a class other than DatasetSpec to build + dataset specifications + ''' + return DatasetSpec + + @classmethod + def link_spec_cls(cls) -> type[LinkSpec]: + ''' The class to use when constructing LinkSpec objects + + Override this if extending to use a class other than LinkSpec to build + link specifications + ''' + return LinkSpec + + @classmethod + def build_spec(cls, spec_dict: dict[str, Any]) -> 'GroupSpec': + ''' Build a specification from a dictionary + + Parameters + ---------- + spec_dict : dict[str, Any] + The dictionary to build the specification from + + Returns + ------- + GroupSpec + The constructed GroupSpec object + ''' + input_dict = spec_dict.copy() + if 'attributes' in input_dict: + input_dict['attributes'] = [AttributeSpec.build_spec(sub_spec) for sub_spec in input_dict['attributes']] + if 'datasets' in input_dict: + input_dict['datasets'] = [ + cls.dataset_spec_cls().build_spec(sub_spec) for sub_spec in input_dict['datasets'] + ] + if 'links' in input_dict: + input_dict['links'] = [cls.link_spec_cls().build_spec(sub_spec) for sub_spec in input_dict['links']] + if 'groups' in input_dict: + input_dict['groups'] = [cls.build_spec(sub_spec) for sub_spec in input_dict['groups']] + return cls(**input_dict) + +# Validation of add_group can happen only after GroupSpec is fully defined +GroupSpec.set_group = validate_call(config=dict(arbitrary_types_allowed=True))(GroupSpec.set_group) +GroupSpec.get_group = validate_call(config=dict(arbitrary_types_allowed=True))(GroupSpec.get_group) +GroupSpec.get_data_type = validate_call(config=dict(arbitrary_types_allowed=True))(GroupSpec.get_data_type) +GroupSpec.resolve_inc_spec = validate_call(config=dict(arbitrary_types_allowed=True))(GroupSpec.resolve_inc_spec) + +# TODO validate call on build_spec throughout diff --git a/src/hdmf/spec/write.py b/src/hdmf/spec/write.py index da2aa97e2..f931e8661 100644 --- a/src/hdmf/spec/write.py +++ b/src/hdmf/spec/write.py @@ -9,7 +9,7 @@ from .catalog import SpecCatalog from .namespace import SpecNamespace -from .spec import GroupSpec, DatasetSpec +from .spec2 import GroupSpec, DatasetSpec from ..utils import docval, getargs, popargs diff --git a/src/hdmf/testing/testcase.py b/src/hdmf/testing/testcase.py index 1be4bcecd..3dfcb2bda 100644 --- a/src/hdmf/testing/testcase.py +++ b/src/hdmf/testing/testcase.py @@ -7,11 +7,10 @@ from .utils import remove_test_file from ..backends.hdf5 import HDF5IO from ..build import Builder -from ..common import validate as common_validate, get_manager from ..container import AbstractContainer, Container, Data from ..utils import get_docval_macro from ..data_utils import AbstractDataChunkIterator - +# NOTE: we do not import ..common here to avoid loading any namespaces unnecessarily class TestCase(unittest.TestCase): """ @@ -271,6 +270,7 @@ def setUpContainer(self): """ def setUp(self): + from ..common import get_manager self.__manager = get_manager() self.container = self.setUpContainer() self.container_type = self.container.__class__.__name__ @@ -322,6 +322,7 @@ def _test_roundtrip(self, read_container, export=False): def roundtripContainer(self, cache_spec=False): """Write the container to an HDF5 file, read the container from the file, and return it.""" + from ..common import get_manager with HDF5IO(self.filename, manager=get_manager(), mode='w') as write_io: write_io.write(self.container, cache_spec=cache_spec) @@ -330,6 +331,7 @@ def roundtripContainer(self, cache_spec=False): def roundtripExportContainer(self, cache_spec=False): """Write the container to an HDF5 file, read it, export it to a new file, read that file, and return it.""" + from ..common import get_manager self.roundtripContainer(cache_spec=cache_spec) HDF5IO.export_io( @@ -343,6 +345,7 @@ def roundtripExportContainer(self, cache_spec=False): def validate(self, experimental=False): """Validate the written and exported files, if they exist.""" + from ..common import get_manager, validate as common_validate if os.path.exists(self.filename): with HDF5IO(self.filename, manager=get_manager(), mode='r') as io: errors = common_validate(io, experimental=experimental) diff --git a/src/hdmf/validate/errors.py b/src/hdmf/validate/errors.py index fb1bfc1b4..42e995894 100644 --- a/src/hdmf/validate/errors.py +++ b/src/hdmf/validate/errors.py @@ -1,6 +1,6 @@ from numpy import dtype -from ..spec.spec import DtypeHelper +from ..spec import DtypeHelper from ..utils import docval, getargs __all__ = [ diff --git a/src/hdmf/validate/validator.py b/src/hdmf/validate/validator.py index 9f5bb927d..40b18daf2 100644 --- a/src/hdmf/validate/validator.py +++ b/src/hdmf/validate/validator.py @@ -8,9 +8,9 @@ from .errors import ExpectedArrayError, IncorrectQuantityError from ..build import GroupBuilder, DatasetBuilder, LinkBuilder, ReferenceBuilder from ..build.builders import BaseBuilder -from ..spec import Spec, AttributeSpec, GroupSpec, DatasetSpec, RefSpec, LinkSpec -from ..spec import SpecNamespace -from ..spec.spec import BaseStorageSpec, DtypeHelper +from ..spec import ( + Spec, AttributeSpec, GroupSpec, DatasetSpec, RefSpec, LinkSpec, BaseStorageSpec, DtypeHelper, SpecNamespace +) from ..utils import docval, getargs, pystr, get_data_shape from ..query import ReferenceResolver diff --git a/tests/unit/spec_tests/test_attribute_spec.py b/tests/unit/spec_tests/test_attribute_spec.py index 807afb5d0..9e470483f 100644 --- a/tests/unit/spec_tests/test_attribute_spec.py +++ b/tests/unit/spec_tests/test_attribute_spec.py @@ -1,4 +1,4 @@ -import json +from pydantic import ValidationError from hdmf.spec import AttributeSpec, RefSpec from hdmf.testing import TestCase @@ -7,100 +7,99 @@ class AttributeSpecTests(TestCase): def test_constructor(self): - spec = AttributeSpec('attribute1', - 'my first attribute', - 'text') - self.assertEqual(spec['name'], 'attribute1') - self.assertEqual(spec['dtype'], 'text') - self.assertEqual(spec['doc'], 'my first attribute') + spec = AttributeSpec(name='attribute1', doc='my first attribute', dtype='text') + self.assertEqual(spec.name, 'attribute1') + self.assertEqual(spec.dtype, 'text') + self.assertEqual(spec.doc, 'my first attribute') self.assertIsNone(spec.parent) - json.dumps(spec) # to ensure there are no circular links + spec_dict = spec.model_dump(exclude_unset=True) + expected = { + 'name': 'attribute1', + 'doc': 'my first attribute', + 'dtype': 'text', + } + self.assertDictEqual(spec_dict, expected) def test_invalid_dtype(self): - with self.assertRaises(ValueError): - AttributeSpec(name='attribute1', - doc='my first attribute', - dtype='invalid' # <-- Invalid dtype must raise a ValueError - ) + with self.assertRaisesRegex(ValueError, r"dtype 'invalid' is not a valid primary data type\."): + AttributeSpec(name='attribute1', doc='my first attribute', dtype='invalid') def test_both_value_and_default_value_set(self): - with self.assertRaises(ValueError): - AttributeSpec(name='attribute1', - doc='my first attribute', - dtype='int', - value=5, - default_value=10 # <-- Default_value and value can't be set at the same time - ) + with self.assertRaisesRegex(ValueError, r"Cannot specify both 'value' and 'default_value'\."): + AttributeSpec( + name='attribute1', + doc='my first attribute', + dtype='int', + value=5, + default_value=10, + ) - def test_colliding_shape_and_dims(self): - with self.assertRaises(ValueError): - AttributeSpec(name='attribute1', - doc='my first attribute', - dtype='int', - dims=['test'], - shape=[None, 2] # <-- Length of shape and dims do not match must raise a ValueError - ) + def test_shape_and_dims_unequal_length(self): + with self.assertRaisesRegex(ValueError, r"'dims' and 'shape' must have the same length\."): + AttributeSpec( + name='attribute1', + doc='my first attribute', + dtype='int', + dims=['test'], + shape=[None, 2], + ) def test_default_value(self): - spec = AttributeSpec('attribute1', - 'my first attribute', - 'text', - default_value='some text') - self.assertEqual(spec['default_value'], 'some text') + spec = AttributeSpec(name='attribute1', doc='my first attribute', dtype='text', default_value='some text') self.assertEqual(spec.default_value, 'some text') def test_shape(self): - shape = [None, 2] - spec = AttributeSpec('attribute1', - 'my first attribute', - 'text', - shape=shape) - self.assertEqual(spec['shape'], shape) - self.assertEqual(spec.shape, shape) + spec = AttributeSpec(name='attribute1', doc='my first attribute', dtype='text', shape=[None, 2]) + self.assertEqual(spec.shape, (None, 2)) self.assertEqual(spec.dims, ('dim_0', 'dim_1')) def test_dims_without_shape(self): - spec = AttributeSpec('attribute1', - 'my first attribute', - 'text', - dims=['test']) - self.assertEqual(spec.shape, (None, )) + spec = AttributeSpec(name='attribute1', doc='my first attribute', dtype='text', dims=['test1', 'test2']) + self.assertEqual(spec.shape, (None, None)) + self.assertEqual(spec.dims, ('test1', 'test2')) + + def test_shape_without_dims(self): + spec = AttributeSpec(name='attribute1', doc='my first attribute', dtype='text', shape=(None, 3)) + self.assertEqual(spec.shape, (None, 3)) + self.assertEqual(spec.dims, ('dim_0', 'dim_1')) def test_build_spec(self): - spec_dict = {'name': 'attribute1', - 'doc': 'my first attribute', - 'dtype': 'text', - 'shape': [None], - 'dims': ['dim1'], - 'value': ['a', 'b']} + spec_dict = { + 'name': 'attribute1', + 'doc': 'my first attribute', + 'dtype': 'text', + 'shape': (None, ), + 'dims': ('dim1',), + 'value': ['a', 'b'] + } ret = AttributeSpec.build_spec(spec_dict) - self.assertTrue(isinstance(ret, AttributeSpec)) - self.assertDictEqual(ret, spec_dict) + self.assertIsInstance(ret, AttributeSpec) + ret_dict = ret.model_dump(exclude_unset=True) + self.assertDictEqual(ret_dict, spec_dict) def test_build_spec_reftype(self): - spec_dict = {'name': 'attribute1', - 'doc': 'my first attribute', - 'dtype': {'target_type': 'AnotherType', 'reftype': 'object'}} - expected = spec_dict.copy() - expected['dtype'] = RefSpec(target_type='AnotherType', reftype='object') + spec_dict = { + 'name': 'attribute1', + 'doc': 'my first attribute', + 'dtype': {'target_type': 'AnotherType', 'reftype': 'object'}, + } ret = AttributeSpec.build_spec(spec_dict) - self.assertTrue(isinstance(ret, AttributeSpec)) - self.assertDictEqual(ret, expected) + self.assertIsInstance(ret, AttributeSpec) + self.assertEqual(ret.dtype, RefSpec(target_type='AnotherType', reftype='object')) + ret_dict = ret.model_dump(exclude_unset=True) + self.assertDictEqual(ret_dict, spec_dict) def test_build_spec_no_doc(self): spec_dict = {'name': 'attribute1', 'dtype': 'text'} - msg = "AttributeSpec.__init__: missing argument 'doc'" - with self.assertRaisesWith(TypeError, msg): + with self.assertRaisesRegex(ValidationError, r'doc\s*Field required'): AttributeSpec.build_spec(spec_dict) - def test_build_warn_extra_args(self): + def test_build_extra_args(self): spec_dict = { 'name': 'attribute1', 'doc': 'test attribute', 'dtype': 'int', 'quantity': '?', } - msg = ("Unexpected keys ['quantity'] in spec {'name': 'attribute1', 'doc': 'test attribute', " - "'dtype': 'int', 'quantity': '?'}") - with self.assertWarnsWith(UserWarning, msg): + with self.assertRaisesRegex(ValidationError, r'quantity\s*Extra inputs are not permitted'): AttributeSpec.build_spec(spec_dict) diff --git a/tests/unit/spec_tests/test_dataset_spec.py b/tests/unit/spec_tests/test_dataset_spec.py index e3b3a93fe..c572de578 100644 --- a/tests/unit/spec_tests/test_dataset_spec.py +++ b/tests/unit/spec_tests/test_dataset_spec.py @@ -1,156 +1,174 @@ -import json +from pydantic import ValidationError -from hdmf.spec import GroupSpec, DatasetSpec, AttributeSpec, DtypeSpec, RefSpec +from hdmf.spec import GroupSpec, DatasetSpec, AttributeSpec, DtypeSpec, RefSpec, QuantityEnum from hdmf.testing import TestCase class DatasetSpecTests(TestCase): def setUp(self): self.attributes = [ - AttributeSpec('attribute1', 'my first attribute', 'text'), - AttributeSpec('attribute2', 'my second attribute', 'text') + AttributeSpec(name='attribute1', dtype='text', doc='my first attribute'), + AttributeSpec(name='attribute2', dtype='text', doc='my second attribute'), ] def test_constructor(self): - spec = DatasetSpec('my first dataset', - 'int', - name='dataset1', - attributes=self.attributes) - self.assertEqual(spec['dtype'], 'int') - self.assertEqual(spec['name'], 'dataset1') - self.assertEqual(spec['doc'], 'my first dataset') - self.assertNotIn('linkable', spec) - self.assertNotIn('data_type_def', spec) - self.assertListEqual(spec['attributes'], self.attributes) - self.assertIs(spec, self.attributes[0].parent) - self.assertIs(spec, self.attributes[1].parent) - json.dumps(spec) - - def test_constructor_datatype(self): - spec = DatasetSpec('my first dataset', - 'int', - name='dataset1', - attributes=self.attributes, - linkable=False, - data_type_def='EphysData') - self.assertEqual(spec['dtype'], 'int') - self.assertEqual(spec['name'], 'dataset1') - self.assertEqual(spec['doc'], 'my first dataset') - self.assertEqual(spec['data_type_def'], 'EphysData') - self.assertFalse(spec['linkable']) - self.assertListEqual(spec['attributes'], self.attributes) - self.assertIs(spec, self.attributes[0].parent) - self.assertIs(spec, self.attributes[1].parent) - - def test_constructor_shape(self): - shape = [None, 2] spec = DatasetSpec( + name='dataset1', + dtype='int', doc='my first dataset', + attributes=self.attributes + ) + self.assertEqual(spec.dtype, 'int') + self.assertEqual(spec.name, 'dataset1') + self.assertEqual(spec.doc, 'my first dataset') + self.assertListEqual(spec.attributes, self.attributes) + spec_dict = spec.model_dump(exclude_unset=True) + expected = { + 'name': 'dataset1', + 'doc': 'my first dataset', + 'dtype': 'int', + 'attributes': [ + { + 'name': 'attribute1', + 'doc': 'my first attribute', + 'dtype': 'text', + }, + { + 'name': 'attribute2', + 'doc': 'my second attribute', + 'dtype': 'text', + }, + ], + } + self.assertDictEqual(spec_dict, expected) + + def test_constructor_data_type_def(self): + spec = DatasetSpec( + data_type_def='EphysData', + name='dataset1', dtype='int', + doc='my first dataset', + attributes=self.attributes, + ) + self.assertEqual(spec.data_type_def, 'EphysData') + + def test_constructor_shape(self): + spec = DatasetSpec( name='dataset1', - shape=shape, + dtype='int', + shape=[None, 2], + doc='my first dataset', ) - self.assertEqual(spec['shape'], shape) - self.assertEqual(spec.shape, shape) + self.assertEqual(spec.shape, (None, 2)) self.assertEqual(spec.dims, ('dim_0', 'dim_1')) def test_dims_without_shape(self): spec = DatasetSpec( - doc='my first dataset', - dtype='int', name='dataset1', + dtype='int', dims=("test",), + doc='my first dataset', ) self.assertEqual(spec.shape, (None, )) - def test_colliding_shape_and_dims(self): - with self.assertRaises(ValueError): + def test_shape_and_dims_unequal_length(self): + with self.assertRaisesRegex(ValueError, r"'dims' and 'shape' must have the same length\."): DatasetSpec( - doc='my first dataset', - dtype='int', name='dataset1', - dims=("test",), + dtype='int', + dims=['test'], shape=[None, 2], + doc='my first dataset', ) - def test_constructor_invalidate_dtype(self): - with self.assertRaises(ValueError): - DatasetSpec(doc='my first dataset', - dtype='my bad dtype', # <-- Expect AssertionError due to bad type - name='dataset1', - dims=(None, None), - attributes=self.attributes, - linkable=False, - data_type_def='EphysData') + def test_constructor_invalid_dtype(self): + with self.assertRaisesRegex(ValueError, r"dtype 'my bad dtype' is not a valid primary data type\."): + DatasetSpec( + data_type_def='EphysData', + name='dataset1', + dtype='my bad dtype', + dims=(None, None), + doc='my first dataset', + attributes=self.attributes, + ) def test_constructor_ref_spec(self): - dtype = RefSpec('TimeSeries', 'object') - spec = DatasetSpec(doc='my first dataset', - dtype=dtype, - name='dataset1', - dims=(None, None), - attributes=self.attributes, - linkable=False, - data_type_def='EphysData') - self.assertDictEqual(spec['dtype'], dtype) - + dtype = RefSpec(target_type='TimeSeries', reftype='object') + spec = DatasetSpec( + data_type_def='EphysData', + name='dataset1', + dtype=dtype, + dims=(None, None), + doc='my first dataset', + attributes=self.attributes, + ) + self.assertEqual(spec.dtype, dtype) def test_constructor_table(self): - dtype1 = DtypeSpec('column1', 'the first column', 'int') - dtype2 = DtypeSpec('column2', 'the second column', 'float') - spec = DatasetSpec('my first table', - [dtype1, dtype2], - name='table1', - attributes=self.attributes) - self.assertEqual(spec['dtype'], [dtype1, dtype2]) - self.assertEqual(spec['name'], 'table1') - self.assertEqual(spec['doc'], 'my first table') - self.assertNotIn('linkable', spec) - self.assertNotIn('data_type_def', spec) - self.assertListEqual(spec['attributes'], self.attributes) - self.assertIs(spec, self.attributes[0].parent) - self.assertIs(spec, self.attributes[1].parent) - json.dumps(spec) + dtype1 = DtypeSpec(name='column1', dtype='int', doc='the first column') + dtype2 = DtypeSpec(name='column2', dtype='float', doc='the second column') + spec = DatasetSpec( + name='table1', + dtype=[dtype1, dtype2], + doc='my first table', + attributes=self.attributes, + ) + self.assertEqual(spec.dtype, [dtype1, dtype2]) def test_constructor_invalid_table(self): - with self.assertRaises(ValueError): - DatasetSpec('my first table', - [DtypeSpec('column1', 'the first column', 'int'), - {} # <--- Bad compound type spec must raise an error - ], - name='table1', - attributes=self.attributes) + with self.assertRaisesRegex(ValidationError, r"5 validation errors for DatasetSpec"): + DatasetSpec( + name='table1', + dtype=[ + DtypeSpec(name='column1', dtype='int', doc='the first column'), + {} # <--- Bad compound type spec must raise an error + ], + doc='my first table', + attributes=self.attributes, + ) def test_constructor_default_value(self): - spec = DatasetSpec(doc='test', - default_value=5, - dtype='int', - data_type_def='test') + spec = DatasetSpec( + data_type_def='test', + dtype='int', + default_value=5, + doc='test', + ) self.assertEqual(spec.default_value, 5) def test_name_with_incompatible_quantity(self): # Check that we raise an error when the quantity allows more than one instance with a fixed name - with self.assertRaises(ValueError): - DatasetSpec(doc='my first dataset', - dtype='int', - name='ds1', - quantity='zero_or_many') - with self.assertRaises(ValueError): - DatasetSpec(doc='my first dataset', - dtype='int', - name='ds1', - quantity='one_or_many') + with self.assertRaisesRegex(ValueError, r"Cannot specify 'name' on a spec that can exist multiple times\."): + DatasetSpec( + name='ds1', + dtype='int', + doc='my first dataset', + quantity='*', + ) + with self.assertRaisesRegex(ValueError, r"Cannot specify 'name' on a spec that can exist multiple times\."): + DatasetSpec( + name='ds1', + dtype='int', + doc='my first dataset', + quantity='+', + ) def test_name_with_compatible_quantity(self): # Make sure compatible quantity flags pass when name is fixed - DatasetSpec(doc='my first dataset', - dtype='int', - name='ds1', - quantity='zero_or_one') - DatasetSpec(doc='my first dataset', - dtype='int', - name='ds1', - quantity=1) + spec = DatasetSpec( + name='ds1', + dtype='int', + doc='my first dataset', + quantity='?', + ) + self.assertEqual(spec.quantity, QuantityEnum.ZERO_OR_ONE) + spec = DatasetSpec( + name='ds1', + dtype='int', + doc='my first dataset', + quantity=1, + ) + self.assertEqual(spec.quantity, 1) def test_data_type_property_value(self): """Test that the property data_type has the expected value""" @@ -163,41 +181,43 @@ def test_data_type_property_value(self): for (data_type_inc, data_type_def), data_type in test_cases.items(): with self.subTest(data_type_inc=data_type_inc, data_type_def=data_type_def, data_type=data_type): - group = GroupSpec('A group', name='group', - data_type_inc=data_type_inc, data_type_def=data_type_def) + group = GroupSpec(data_type_def=data_type_def, data_type_inc=data_type_inc, + name='group', doc='A group') self.assertEqual(group.data_type, data_type) def test_constructor_value(self): - spec = DatasetSpec(doc='my first dataset', dtype='int', name='dataset1', value=42) + spec = DatasetSpec( + name='dataset1', + dtype='int', + value=42, + doc='my first dataset', + ) assert spec.value == 42 - def test_build_warn_extra_args(self): + def test_build_extra_args(self): spec_dict = { 'name': 'dataset1', 'doc': 'test dataset', 'dtype': 'int', 'required': True, } - msg = ("Unexpected keys ['required'] in spec {'name': 'dataset1', 'doc': 'test dataset', " - "'dtype': 'int', 'required': True}") - with self.assertWarnsWith(UserWarning, msg): + # TODO + with self.assertRaisesRegex(ValidationError, r'required\s*Extra inputs are not permitted'): DatasetSpec.build_spec(spec_dict) def test_constructor_validates_name(self): - with self.assertRaisesWith( - ValueError, - "Name 'one/two' is invalid. Names of Groups and Datasets cannot contain '/'", - ): - DatasetSpec(doc='my first dataset', dtype='int', name='one/two') + with self.assertRaisesRegex(ValueError, r"Invalid character '/' in 'name'\."): + DatasetSpec( + name='one/two', + dtype='int', + doc='my first dataset', + ) def test_constructor_validates_default_name(self): - with self.assertRaisesWith( - ValueError, - "Default name 'one/two' is invalid. Names of Groups and Datasets cannot contain '/'", - ): - DatasetSpec(doc='my first dataset', dtype='int', default_name='one/two', data_type_def='test') - - def test_constructor_value_default_value(self): - msg = "cannot specify 'value' and 'default_value'" - with self.assertRaisesWith(ValueError, msg): - DatasetSpec(doc='my first dataset', dtype='int', name='dataset1', value=42, default_value=0) + with self.assertRaisesRegex(ValueError, r"Invalid character '/' in 'default_name'\."): + DatasetSpec( + data_type_def='test', + default_name='one/two', + dtype='int', + doc='my first dataset', + ) diff --git a/tests/unit/spec_tests/test_dtype_spec.py b/tests/unit/spec_tests/test_dtype_spec.py index 300e7ecc6..d874bbbc5 100644 --- a/tests/unit/spec_tests/test_dtype_spec.py +++ b/tests/unit/spec_tests/test_dtype_spec.py @@ -3,8 +3,6 @@ class DtypeSpecHelper(TestCase): - def setUp(self): - pass def test_recommended_dtypes(self): self.assertListEqual(DtypeHelper.recommended_primary_dtypes, @@ -16,53 +14,53 @@ def test_valid_primary_dtypes(self): self.assertSetEqual(a, DtypeHelper.valid_primary_dtypes) def test_simplify_cpd_type(self): - compound_type = [DtypeSpec('test', 'test field', 'float'), - DtypeSpec('test2', 'test field2', 'int')] + compound_type = [ + DtypeSpec(name='test', doc='test field', dtype='float'), + DtypeSpec(name='test2', doc='test field2', dtype='int'), + ] expected_result = ['float', 'int'] result = DtypeHelper.simplify_cpd_type(compound_type) self.assertListEqual(result, expected_result) def test_simplify_cpd_type_ref(self): - compound_type = [DtypeSpec('test', 'test field', 'float'), - DtypeSpec('test2', 'test field2', RefSpec(target_type='MyType', reftype='object'))] + compound_type = [ + DtypeSpec(name='test', doc='test field', dtype='float'), + DtypeSpec(name='test2', doc='test field2', dtype=RefSpec(target_type='MyType', reftype='object')), + ] expected_result = ['float', 'object'] result = DtypeHelper.simplify_cpd_type(compound_type) self.assertListEqual(result, expected_result) def test_check_dtype_ok(self): - self.assertEqual('int', DtypeHelper.check_dtype('int')) + DtypeHelper.check_dtype(dtype='int') # should not raise def test_check_dtype_bad(self): - msg = "dtype 'bad dtype' is not a valid primary data type." - with self.assertRaisesRegex(ValueError, msg): - DtypeHelper.check_dtype('bad dtype') + with self.assertRaisesRegex(ValueError, r"'bad dtype' is not a valid primary data type\."): + DtypeHelper.check_dtype(dtype='bad dtype') def test_check_dtype_ref(self): refspec = RefSpec(target_type='target', reftype='object') - self.assertIs(refspec, DtypeHelper.check_dtype(refspec)) + DtypeHelper.check_dtype(dtype=refspec) # should not raise def test_is_allowed(self): - self.assertTrue(DtypeHelper.is_allowed_dtype('int32', 'int')) - self.assertTrue(DtypeHelper.is_allowed_dtype('float64', 'float')) - self.assertFalse(DtypeHelper.is_allowed_dtype('int32', 'float')) - self.assertFalse(DtypeHelper.is_allowed_dtype('string', 'int')) - self.assertTrue(DtypeHelper.is_allowed_dtype('object', 'object')) - self.assertTrue(DtypeHelper.is_allowed_dtype('int64', 'numeric')) - self.assertTrue(DtypeHelper.is_allowed_dtype('float32', 'numeric')) - self.assertFalse(DtypeHelper.is_allowed_dtype('string', 'numeric')) - self.assertTrue(DtypeHelper.is_allowed_dtype('numeric', 'numeric')) + self.assertTrue(DtypeHelper.is_allowed_dtype(new='int32', orig='int')) + self.assertTrue(DtypeHelper.is_allowed_dtype(new='float64', orig='float')) + self.assertFalse(DtypeHelper.is_allowed_dtype(new='int32', orig='float')) + self.assertFalse(DtypeHelper.is_allowed_dtype(new='string', orig='int')) + self.assertTrue(DtypeHelper.is_allowed_dtype(new='object', orig='object')) + self.assertTrue(DtypeHelper.is_allowed_dtype(new='int64', orig='numeric')) + self.assertTrue(DtypeHelper.is_allowed_dtype(new='float32', orig='numeric')) + self.assertFalse(DtypeHelper.is_allowed_dtype(new='string', orig='numeric')) + self.assertTrue(DtypeHelper.is_allowed_dtype(new='numeric', orig='numeric')) - msg = "Unknown dtype 'bad dtype'" - with self.assertRaisesRegex(ValueError, msg): - DtypeHelper.is_allowed_dtype('int32', 'bad dtype') + with self.assertRaisesRegex(ValueError, r"Invalid dtype 'bad dtype'\."): + DtypeHelper.is_allowed_dtype(new='int32', orig='bad dtype') class DtypeSpecTests(TestCase): - def setUp(self): - pass def test_constructor(self): - spec = DtypeSpec('column1', 'an example column', 'int') + spec = DtypeSpec(name='column1', doc='an example column', dtype='int') self.assertEqual(spec.doc, 'an example column') self.assertEqual(spec.name, 'column1') self.assertEqual(spec.dtype, 'int') @@ -73,23 +71,10 @@ def test_build_spec(self): self.assertEqual(spec.name, 'column1') self.assertEqual(spec.dtype, 'int') - def test_invalid_refspec_dict(self): - """Test missing or bad target key for RefSpec.""" - msg = "'dtype' must have the key 'target_type'" - with self.assertRaisesWith(ValueError, msg): - DtypeSpec.assertValidDtype({'no target': 'test', 'reftype': 'object'}) - def test_refspec_dtype(self): - # just making sure this does not cause an error - DtypeSpec('column1', 'an example column', RefSpec('TimeSeries', 'object')) + # make sure this does not cause an error + DtypeSpec(name='column1', doc='an example column', dtype=RefSpec(target_type='TimeSeries', reftype='object')) def test_invalid_dtype(self): - msg = "dtype 'bad dtype' is not a valid primary data type." - with self.assertRaisesRegex(ValueError, msg): - DtypeSpec('column1', 'an example column', dtype='bad dtype') - - def test_is_ref(self): - spec = DtypeSpec('column1', 'an example column', RefSpec('TimeSeries', 'object')) - self.assertTrue(DtypeSpec.is_ref(spec)) - spec = DtypeSpec('column1', 'an example column', 'int') - self.assertFalse(DtypeSpec.is_ref(spec)) + with self.assertRaisesRegex(ValueError, r"dtype 'bad dtype' is not a valid primary data type\."): + DtypeSpec(name='column1', doc='an example column', dtype='bad dtype') diff --git a/tests/unit/spec_tests/test_group_spec.py b/tests/unit/spec_tests/test_group_spec.py index 73d42ae13..0b9b70af8 100644 --- a/tests/unit/spec_tests/test_group_spec.py +++ b/tests/unit/spec_tests/test_group_spec.py @@ -1,4 +1,4 @@ -import json +from pydantic import ValidationError from hdmf.spec import GroupSpec, DatasetSpec, AttributeSpec, LinkSpec from hdmf.testing import TestCase @@ -7,145 +7,201 @@ class GroupSpecTests(TestCase): def setUp(self): self.attributes = [ - AttributeSpec('attribute1', 'my first attribute', 'text'), - AttributeSpec('attribute2', 'my second attribute', 'text') + AttributeSpec(name='attribute1', dtype='text', doc='my first attribute'), + AttributeSpec(name='attribute2', dtype='text', doc='my second attribute') ] self.dset1_attributes = [ - AttributeSpec('attribute3', 'my third attribute', 'text'), - AttributeSpec('attribute4', 'my fourth attribute', 'text') + AttributeSpec(name='attribute3', dtype='text', doc='my third attribute'), + AttributeSpec(name='attribute4', dtype='text', doc='my fourth attribute') ] self.dset2_attributes = [ - AttributeSpec('attribute5', 'my fifth attribute', 'text'), - AttributeSpec('attribute6', 'my sixth attribute', 'text') + AttributeSpec(name='attribute5', dtype='text', doc='my fifth attribute'), + AttributeSpec(name='attribute6', dtype='text', doc='my sixth attribute') ] self.datasets = [ - DatasetSpec('my first dataset', - 'int', - name='dataset1', - attributes=self.dset1_attributes, - linkable=True), - DatasetSpec('my second dataset', - 'int', - name='dataset2', - attributes=self.dset2_attributes, - linkable=True, - data_type_def='VoltageArray') + DatasetSpec( + name='dataset1', + dtype='int', + doc='my first dataset', + attributes=self.dset1_attributes, + ), + DatasetSpec( + data_type_def='VoltageArray', + name='dataset2', + dtype='int', + doc='my second dataset', + attributes=self.dset2_attributes, + ) ] self.subgroups = [ - GroupSpec('A test subgroup', - name='subgroup1', - linkable=False), - GroupSpec('A test subgroup', - name='subgroup2', - linkable=False) + GroupSpec(name='subgroup1', doc='A test subgroup'), + GroupSpec(name='subgroup2', doc='Another test subgroup'), ] def test_constructor(self): - spec = GroupSpec('A test group', - name='root_constructor', - groups=self.subgroups, - datasets=self.datasets, - attributes=self.attributes, - linkable=False) - self.assertFalse(spec['linkable']) - self.assertListEqual(spec['attributes'], self.attributes) - self.assertListEqual(spec['datasets'], self.datasets) - self.assertNotIn('data_type_def', spec) + spec = GroupSpec( + name='root_constructor', + doc='A test group', + attributes=self.attributes, + datasets=self.datasets, + groups=self.subgroups, + ) + self.assertListEqual(spec.attributes, self.attributes) + self.assertListEqual(spec.datasets, self.datasets) + self.assertListEqual(spec.groups, self.subgroups) self.assertIs(spec, self.subgroups[0].parent) self.assertIs(spec, self.subgroups[1].parent) self.assertIs(spec, self.attributes[0].parent) self.assertIs(spec, self.attributes[1].parent) self.assertIs(spec, self.datasets[0].parent) self.assertIs(spec, self.datasets[1].parent) - json.dumps(spec) + self.maxDiff = None + spec_dict = spec.model_dump(exclude_unset=True) + expected = { + 'name': 'root_constructor', + 'doc': 'A test group', + 'groups': [ + { + 'name': 'subgroup1', + 'doc': 'A test subgroup', + }, + { + 'name': 'subgroup2', + 'doc': 'Another test subgroup', + } + ], + 'datasets': [ + { + 'name': 'dataset1', + 'doc': 'my first dataset', + 'dtype': 'int', + 'attributes': [ + { + 'name': 'attribute3', + 'doc': 'my third attribute', + 'dtype': 'text', + }, + { + 'name': 'attribute4', + 'dtype': 'text', + 'doc': 'my fourth attribute', + } + ], + }, + { + 'name': 'dataset2', + 'dtype': 'int', + 'doc': 'my second dataset', + 'attributes': [ + { + 'name': 'attribute5', + 'dtype': 'text', + 'doc': 'my fifth attribute', + }, + { + 'name': 'attribute6', + 'dtype': 'text', + 'doc': 'my sixth attribute', + } + ], + 'data_type_def': 'VoltageArray' + } + ], + 'attributes': [ + { + 'name': 'attribute1', + 'dtype': 'text', + 'doc': 'my first attribute', + }, + { + 'name': 'attribute2', + 'dtype': 'text', + 'doc': 'my second attribute', + } + ], + } + self.assertDictEqual(spec_dict, expected) def test_constructor_datatype(self): - spec = GroupSpec('A test group', - name='root_constructor_datatype', - datasets=self.datasets, - attributes=self.attributes, - linkable=False, - data_type_def='EphysData') - self.assertFalse(spec['linkable']) - self.assertListEqual(spec['attributes'], self.attributes) - self.assertListEqual(spec['datasets'], self.datasets) - self.assertEqual(spec['data_type_def'], 'EphysData') + spec = GroupSpec( + data_type_def='EphysData', + name='root_constructor_datatype', + doc='A test group', + attributes=self.attributes, + datasets=self.datasets, + ) + self.assertListEqual(spec.attributes, self.attributes) + self.assertListEqual(spec.datasets, self.datasets) + self.assertEqual(spec.data_type_def, 'EphysData') self.assertIs(spec, self.attributes[0].parent) self.assertIs(spec, self.attributes[1].parent) self.assertIs(spec, self.datasets[0].parent) self.assertIs(spec, self.datasets[1].parent) self.assertEqual(spec.data_type_def, 'EphysData') self.assertIsNone(spec.data_type_inc) - json.dumps(spec) + spec_dict = spec.model_dump(exclude_unset=True) + self.assertEqual(spec_dict.get('data_type_def'), 'EphysData') def test_set_parent_exists(self): - GroupSpec('A test group', - name='root_constructor', - groups=self.subgroups) - msg = 'Cannot re-assign parent.' - with self.assertRaisesWith(AttributeError, msg): + GroupSpec(doc='A test group', name='root_constructor', groups=self.subgroups) + with self.assertRaisesWith(ValueError, 'Parent cannot be changed after being set.'): self.subgroups[0].parent = self.subgroups[1] def test_set_dataset(self): - spec = GroupSpec('A test group', - name='root_test_set_dataset', - linkable=False, - data_type_def='EphysData') + spec = GroupSpec( + data_type_def='EphysData', + name='root_test_set_dataset', + doc='A test group', + ) spec.set_dataset(self.datasets[0]) self.assertIs(spec, self.datasets[0].parent) def test_set_link(self): group = GroupSpec( + name='root', doc='A test group', - name='root' ) link = LinkSpec( - doc='A test link', + name='link_name', target_type='LinkTarget', - name='link_name' + doc='A test link', ) group.set_link(link) self.assertIs(group, link.parent) self.assertIs(group.get_link('link_name'), link) - def test_add_link(self): - group = GroupSpec( + def test_set_group(self): + spec = GroupSpec( + data_type_def='EphysData', + name='root_test_set_group', doc='A test group', - name='root' - ) - group.add_link( - 'A test link', - 'LinkTarget', - name='link_name' ) - self.assertIsInstance(group.get_link('link_name'), LinkSpec) - - def test_set_group(self): - spec = GroupSpec('A test group', - name='root_test_set_group', - linkable=False, - data_type_def='EphysData') spec.set_group(self.subgroups[0]) spec.set_group(self.subgroups[1]) - self.assertListEqual(spec['groups'], self.subgroups) + self.assertListEqual(spec.groups, self.subgroups) self.assertIs(spec, self.subgroups[0].parent) self.assertIs(spec, self.subgroups[1].parent) - json.dumps(spec) - - def test_add_group(self): - group = GroupSpec( - doc='A test group', - name='root' - ) - group.add_group( - 'A test group', - name='subgroup' - ) - self.assertIsInstance(group.get_group('subgroup'), GroupSpec) + spec_dict = spec.model_dump(exclude_unset=True) + expected = { + 'data_type_def': 'EphysData', + 'name': 'root_test_set_group', + 'doc': 'A test group', + 'groups': [ + { + 'name': 'subgroup1', + 'doc': 'A test subgroup', + }, + { + 'name': 'subgroup2', + 'doc': 'Another test subgroup', + } + ], + } + self.assertDictEqual(spec_dict, expected) def assertDatasetsEqual(self, spec1, spec2): @@ -166,74 +222,58 @@ def assertAttributesEqual(self, spec1, spec2): for i in range(len(spec1_attr)): self.assertDictEqual(spec1_attr[i], spec2_attr[i]) - def test_add_attribute(self): - spec = GroupSpec('A test group', - name='root_constructor', - groups=self.subgroups, - datasets=self.datasets, - linkable=False) - for attrspec in self.attributes: - spec.add_attribute(**attrspec) - self.assertListEqual(spec['attributes'], self.attributes) - self.assertListEqual(spec['datasets'], self.datasets) - self.assertNotIn('data_type_def', spec) - self.assertIs(spec, self.subgroups[0].parent) - self.assertIs(spec, self.subgroups[1].parent) - self.assertIs(spec, spec.attributes[0].parent) - self.assertIs(spec, spec.attributes[1].parent) - self.assertIs(spec, self.datasets[0].parent) - self.assertIs(spec, self.datasets[1].parent) - json.dumps(spec) - def test_update_attribute_spec(self): - spec = GroupSpec('A test group', - name='root_constructor', - attributes=[AttributeSpec('attribute1', 'my first attribute', 'text'), - AttributeSpec('attribute2', 'my second attribute', 'text')]) - spec.set_attribute(AttributeSpec('attribute2', 'my second attribute', 'int', value=5)) - res = spec.get_attribute('attribute2') - self.assertEqual(res.value, 5) - self.assertEqual(res.dtype, 'int') + spec = GroupSpec( + name='root_constructor', + doc='A test group', + attributes=[ + AttributeSpec(name='attribute1', dtype='text', doc='my first attribute'), + AttributeSpec(name='attribute2', dtype='text', doc='my second attribute'), + ], + ) + with self.assertRaisesWith(ValueError, "Attribute 'attribute2' already exists in spec 'root_constructor'."): + spec.set_attribute(AttributeSpec(name='attribute2', dtype='int', value=5, doc='my second attribute')) def test_path(self): - GroupSpec('A test group', - name='root_constructor', - groups=self.subgroups, - datasets=self.datasets, - attributes=self.attributes, - linkable=False) + GroupSpec( + name='root_constructor', + doc='A test group', + attributes=self.attributes, + datasets=self.datasets, + groups=self.subgroups, + ) self.assertEqual(self.attributes[0].path, 'root_constructor/attribute1') self.assertEqual(self.datasets[0].path, 'root_constructor/dataset1') self.assertEqual(self.subgroups[0].path, 'root_constructor/subgroup1') def test_path_complicated(self): - attribute = AttributeSpec('attribute1', 'my fifth attribute', 'text') - dataset = DatasetSpec('my first dataset', - 'int', - name='dataset1', + attribute = AttributeSpec(name='attribute1', dtype='text', doc='my fifth attribute') + dataset = DatasetSpec(name='dataset1', + dtype='int', + doc='my first dataset', attributes=[attribute]) - subgroup = GroupSpec('A subgroup', - name='subgroup1', + subgroup = GroupSpec(name='subgroup1', + doc='A subgroup', datasets=[dataset]) self.assertEqual(attribute.path, 'subgroup1/dataset1/attribute1') - _ = GroupSpec('A test group', - name='root', + _ = GroupSpec(name='root', + doc='A test group', groups=[subgroup]) self.assertEqual(attribute.path, 'root/subgroup1/dataset1/attribute1') def test_path_no_name(self): - attribute = AttributeSpec('attribute1', 'my fifth attribute', 'text') - dataset = DatasetSpec('my first dataset', - 'int', - data_type_inc='DatasetType', + attribute = AttributeSpec(name='attribute1', dtype='text', doc='my fifth attribute') + dataset = DatasetSpec(data_type_inc='DatasetType', + dtype='int', + doc='my first dataset', attributes=[attribute]) - subgroup = GroupSpec('A subgroup', - data_type_def='GroupType', + subgroup = GroupSpec(data_type_def='GroupType', + doc='A subgroup', datasets=[dataset]) - _ = GroupSpec('A test group', - name='root', + _ = GroupSpec(name='root', + doc='A test group', groups=[subgroup]) self.assertEqual(attribute.path, 'root/GroupType/DatasetType/attribute1') @@ -249,288 +289,293 @@ def test_data_type_property_value(self): for (data_type_inc, data_type_def), data_type in test_cases.items(): with self.subTest(data_type_inc=data_type_inc, data_type_def=data_type_def, data_type=data_type): - dataset = DatasetSpec('A dataset', 'int', name='dataset', - data_type_inc=data_type_inc, data_type_def=data_type_def) + dataset = DatasetSpec(data_type_def=data_type_def, data_type_inc=data_type_inc, + name='dataset', dtype='int', doc='A dataset') self.assertEqual(dataset.data_type, data_type) - def test_get_data_type_spec(self): - expected = AttributeSpec('data_type', 'the data type of this object', 'text', value='MyType') - self.assertDictEqual(GroupSpec.get_data_type_spec('MyType'), expected) - - def test_get_namespace_spec(self): - expected = AttributeSpec('namespace', 'the namespace for the data type of this object', 'text', required=False) - self.assertDictEqual(GroupSpec.get_namespace_spec(), expected) - def test_build_warn_extra_args(self): spec_dict = { 'name': 'group1', 'doc': 'test group', 'required': True, } - msg = "Unexpected keys ['required'] in spec {'name': 'group1', 'doc': 'test group', 'required': True}" - with self.assertWarnsWith(UserWarning, msg): + with self.assertRaisesRegex(ValidationError, r'required\s*Extra inputs are not permitted'): GroupSpec.build_spec(spec_dict) class TestNotAllowedConfig(TestCase): def test_no_name_no_def_no_inc(self): - msg = ("Cannot create Group or Dataset spec with no name without specifying 'data_type_def' " - "and/or 'data_type_inc'.") - with self.assertRaisesWith(ValueError, msg): + msg = r"At least one of 'name', 'data_type_def', or 'data_type_inc' must be specified\." + with self.assertRaisesRegex(ValidationError, msg): GroupSpec(doc='A test group') def test_name_with_multiple(self): - msg = ("Cannot give specific name to something that can exist multiple times: name='MyGroup', quantity='*'") - with self.assertRaisesWith(ValueError, msg): - GroupSpec(doc='A test group', name='MyGroup', quantity='*') + msg = r"Cannot specify 'name' on a spec that can exist multiple times\." + with self.assertRaisesRegex(ValidationError, msg): + GroupSpec(name='MyGroup', doc='A test group', quantity='*') def test_same_data_type_def_inc(self): - msg = ("data_type_inc and data_type_def cannot be the same: MyType. Ignoring data_type_inc.") - with self.assertWarnsWith(UserWarning, msg): - GroupSpec(doc='A test group', data_type_def='MyType', data_type_inc='MyType') + msg = r"Cannot specify the same value for 'data_type_def' and 'data_type_inc': MyType" + with self.assertRaisesRegex(ValueError, msg): + GroupSpec(data_type_def='MyType', data_type_inc='MyType', doc='A test group') class GroupSpecWithLinksTest(TestCase): def test_constructor(self): - link0 = LinkSpec(doc='Link 0', target_type='TargetType0') - link1 = LinkSpec(doc='Link 1', target_type='TargetType1') + link0 = LinkSpec(target_type='TargetType0', doc='Link 0') + link1 = LinkSpec(target_type='TargetType1', doc='Link 1') links = [link0, link1] spec = GroupSpec( - doc='A test group', name='root', + doc='A test group', links=links ) self.assertIs(spec, links[0].parent) self.assertIs(spec, links[1].parent) - json.dumps(spec) + spec_dict = spec.model_dump(exclude_unset=True) + expected = { + 'name': 'root', + 'doc': 'A test group', + 'links': [ + { + 'target_type': 'TargetType0', + 'doc': 'Link 0', + }, + { + 'target_type': 'TargetType1', + 'doc': 'Link 1', + } + ], + } + self.assertDictEqual(spec_dict, expected) class SpecWithDupsTest(TestCase): - def test_two_unnamed_group_same_type(self): + def test_two_unnamed_groups_same_type(self): """Test creating a group contains multiple unnamed groups with type X.""" - child0 = GroupSpec(doc='Group 0', data_type_inc='Type0') - child1 = GroupSpec(doc='Group 1', data_type_inc='Type0') - msg = "Cannot have multiple groups/datasets with the same data type without specifying name" - with self.assertRaisesWith(ValueError, msg): + child0 = GroupSpec(data_type_inc='Type0', doc='Group 0') + child1 = GroupSpec(data_type_inc='Type0', doc='Group 1') + msg = r"Duplicate data_type: 'Type0' is used in multiple unnamed subspecs in group 'parent'\." + with self.assertRaisesRegex(ValidationError, msg): GroupSpec( - doc='A test group', + data_type_def='ParentType', name='parent', + doc='A test group', groups=[child0, child1], - data_type_def='ParentType' ) - def test_named_unnamed_group_with_def_same_type(self): + def test_named_unnamed_groups_with_def_same_type(self): """Test get_data_type when a group contains both a named and unnamed group with type X.""" - child0 = GroupSpec(doc='Group 0', data_type_def='Type0', name='type0') - child1 = GroupSpec(doc='Group 1', data_type_inc='Type0') + child0 = GroupSpec(data_type_def='Type0', name='type0', doc='Group 0') + child1 = GroupSpec(data_type_inc='Type0', doc='Group 1') parent_spec = GroupSpec( - doc='A test group', + data_type_def='ParentType', name='parent', + doc='A test group', groups=[child0, child1], - data_type_def='ParentType' ) self.assertIs(parent_spec.get_data_type('Type0'), child1) - def test_named_unnamed_group_same_type(self): + def test_named_unnamed_groups_same_type(self): """Test get_data_type when a group contains both a named and unnamed group with type X.""" - child0 = GroupSpec(doc='Group 0', data_type_inc='Type0', name='type0') - child1 = GroupSpec(doc='Group 1', data_type_inc='Type0', name='type1') - child2 = GroupSpec(doc='Group 2', data_type_inc='Type0') + child0 = GroupSpec(data_type_inc='Type0', name='type0', doc='Group 0') + child1 = GroupSpec(data_type_inc='Type0', name='type1', doc='Group 1') + child2 = GroupSpec(data_type_inc='Type0', doc='Group 2') parent_spec = GroupSpec( - doc='A test group', + data_type_def='ParentType', name='parent', + doc='A test group', groups=[child0, child1, child2], - data_type_def='ParentType' ) self.assertIs(parent_spec.get_data_type('Type0'), child2) - def test_unnamed_named_group_same_type(self): + def test_unnamed_named_groups_same_type(self): """Test get_data_type when a group contains both an unnamed and named group with type X.""" - child0 = GroupSpec(doc='Group 0', data_type_inc='Type0') - child1 = GroupSpec(doc='Group 1', data_type_inc='Type0', name='type1') + child0 = GroupSpec(data_type_inc='Type0', doc='Group 0') + child1 = GroupSpec(data_type_inc='Type0', name='type1', doc='Group 1') parent_spec = GroupSpec( - doc='A test group', + data_type_def='ParentType', name='parent', + doc='A test group', groups=[child0, child1], - data_type_def='ParentType' ) self.assertIs(parent_spec.get_data_type('Type0'), child0) - def test_two_named_group_same_type(self): + def test_two_named_groups_same_type(self): """Test get_data_type when a group contains multiple named groups with type X.""" - child0 = GroupSpec(doc='Group 0', data_type_inc='Type0', name='group0') - child1 = GroupSpec(doc='Group 1', data_type_inc='Type0', name='group1') + child0 = GroupSpec(data_type_inc='Type0', name='group0', doc='Group 0') + child1 = GroupSpec(data_type_inc='Type0', name='group1', doc='Group 1') parent_spec = GroupSpec( - doc='A test group', + data_type_def='ParentType', name='parent', + doc='A test group', groups=[child0, child1], - data_type_def='ParentType' ) self.assertEqual(parent_spec.get_data_type('Type0'), [child0, child1]) def test_two_unnamed_datasets_same_type(self): """Test creating a group contains multiple unnamed datasets with type X.""" - child0 = DatasetSpec(doc='Group 0', data_type_inc='Type0') - child1 = DatasetSpec(doc='Group 1', data_type_inc='Type0') - msg = "Cannot have multiple groups/datasets with the same data type without specifying name" - with self.assertRaisesWith(ValueError, msg): + child0 = DatasetSpec(data_type_inc='Type0', doc='Group 0') + child1 = DatasetSpec(data_type_inc='Type0', doc='Group 1') + msg = r"Duplicate data_type: 'Type0' is used in multiple unnamed subspecs in group 'parent'\." + with self.assertRaisesRegex(ValidationError, msg): GroupSpec( - doc='A test group', + data_type_def='ParentType', name='parent', + doc='A test group', datasets=[child0, child1], - data_type_def='ParentType' ) - def test_named_unnamed_dataset_with_def_same_type(self): + def test_named_unnamed_datasets_with_def_same_type(self): """Test get_data_type when a group contains both a named and unnamed dataset with type X.""" - child0 = DatasetSpec(doc='Group 0', data_type_def='Type0', name='type0') - child1 = DatasetSpec(doc='Group 1', data_type_inc='Type0') + child0 = DatasetSpec(data_type_def='Type0', name='type0', doc='Group 0') + child1 = DatasetSpec(data_type_inc='Type0', doc='Group 1') parent_spec = GroupSpec( - doc='A test group', + data_type_def='ParentType', name='parent', + doc='A test group', datasets=[child0, child1], - data_type_def='ParentType' ) self.assertIs(parent_spec.get_data_type('Type0'), child1) def test_named_unnamed_dataset_same_type(self): """Test get_data_type when a group contains both a named and unnamed dataset with type X.""" - child0 = DatasetSpec(doc='Group 0', data_type_inc='Type0', name='type0') - child1 = DatasetSpec(doc='Group 1', data_type_inc='Type0') + child0 = DatasetSpec(data_type_inc='Type0', name='type0', doc='Group 0') + child1 = DatasetSpec(data_type_inc='Type0', doc='Group 1') parent_spec = GroupSpec( - doc='A test group', + data_type_def='ParentType', name='parent', + doc='A test group', datasets=[child0, child1], - data_type_def='ParentType' ) self.assertIs(parent_spec.get_data_type('Type0'), child1) def test_two_named_unnamed_dataset_same_type(self): """Test get_data_type when a group contains two named and one unnamed dataset with type X.""" - child0 = DatasetSpec(doc='Group 0', data_type_inc='Type0', name='type0') - child1 = DatasetSpec(doc='Group 1', data_type_inc='Type0', name='type1') - child2 = DatasetSpec(doc='Group 2', data_type_inc='Type0') + child0 = DatasetSpec(data_type_inc='Type0', name='type0', doc='Group 0') + child1 = DatasetSpec(data_type_inc='Type0', name='type1', doc='Group 1') + child2 = DatasetSpec(data_type_inc='Type0', doc='Group 2') parent_spec = GroupSpec( - doc='A test group', + data_type_def='ParentType', name='parent', + doc='A test group', datasets=[child0, child1, child2], - data_type_def='ParentType' ) self.assertIs(parent_spec.get_data_type('Type0'), child2) def test_unnamed_named_dataset_same_type(self): """Test get_data_type when a group contains both an unnamed and named dataset with type X.""" - child0 = DatasetSpec(doc='Group 0', data_type_inc='Type0') - child1 = DatasetSpec(doc='Group 1', data_type_inc='Type0', name='type1') + child0 = DatasetSpec(data_type_inc='Type0', doc='Group 0') + child1 = DatasetSpec(data_type_inc='Type0', name='type1', doc='Group 1') parent_spec = GroupSpec( - doc='A test group', + data_type_def='ParentType', name='parent', + doc='A test group', datasets=[child0, child1], - data_type_def='ParentType' ) self.assertIs(parent_spec.get_data_type('Type0'), child0) def test_two_named_datasets_same_type(self): """Test get_data_type when a group contains multiple named datasets with type X.""" - child0 = DatasetSpec(doc='Group 0', data_type_inc='Type0', name='group0') - child1 = DatasetSpec(doc='Group 1', data_type_inc='Type0', name='group1') + child0 = DatasetSpec(data_type_inc='Type0', name='group0', doc='Group 0') + child1 = DatasetSpec(data_type_inc='Type0', name='group1', doc='Group 1') parent_spec = GroupSpec( - doc='A test group', + data_type_def='ParentType', name='parent', + doc='A test group', datasets=[child0, child1], - data_type_def='ParentType' ) self.assertEqual(parent_spec.get_data_type('Type0'), [child0, child1]) def test_three_named_datasets_same_type(self): """Test get_target_type when a group contains three named links with type X.""" - child0 = DatasetSpec(doc='Group 0', data_type_inc='Type0', name='group0') - child1 = DatasetSpec(doc='Group 1', data_type_inc='Type0', name='group1') - child2 = DatasetSpec(doc='Group 2', data_type_inc='Type0', name='group2') + child0 = DatasetSpec(data_type_inc='Type0', name='group0', doc='Group 0') + child1 = DatasetSpec(data_type_inc='Type0', name='group1', doc='Group 1') + child2 = DatasetSpec(data_type_inc='Type0', name='group2', doc='Group 2') parent_spec = GroupSpec( - doc='A test group', + data_type_def='ParentType', name='parent', + doc='A test group', datasets=[child0, child1, child2], - data_type_def='ParentType' ) self.assertEqual(parent_spec.get_data_type('Type0'), [child0, child1, child2]) def test_two_unnamed_links_same_type(self): """Test creating a group contains multiple unnamed links with type X.""" - child0 = LinkSpec(doc='Group 0', target_type='Type0') - child1 = LinkSpec(doc='Group 1', target_type='Type0') - msg = "Cannot have multiple links with the same target type without specifying name" - with self.assertRaisesWith(ValueError, msg): + child0 = LinkSpec(target_type='Type0', doc='Group 0') + child1 = LinkSpec(target_type='Type0', doc='Group 1') + msg = r"Duplicate target_type: 'Type0' is used in multiple unnamed links in group 'parent'\." + with self.assertRaisesRegex(ValueError, msg): GroupSpec( - doc='A test group', + data_type_def='ParentType', name='parent', + doc='A test group', links=[child0, child1], - data_type_def='ParentType' ) def test_named_unnamed_link_same_type(self): """Test get_target_type when a group contains both a named and unnamed link with type X.""" - child0 = LinkSpec(doc='Group 0', target_type='Type0', name='type0') - child1 = LinkSpec(doc='Group 1', target_type='Type0') + child0 = LinkSpec(name='type0', target_type='Type0', doc='Group 0') + child1 = LinkSpec(target_type='Type0', doc='Group 1') parent_spec = GroupSpec( - doc='A test group', + data_type_def='ParentType', name='parent', + doc='A test group', links=[child0, child1], - data_type_def='ParentType' ) self.assertIs(parent_spec.get_target_type('Type0'), child1) def test_two_named_unnamed_link_same_type(self): """Test get_target_type when a group contains two named and one unnamed link with type X.""" - child0 = LinkSpec(doc='Group 0', target_type='Type0', name='type0') - child1 = LinkSpec(doc='Group 1', target_type='Type0', name='type1') - child2 = LinkSpec(doc='Group 2', target_type='Type0') + child0 = LinkSpec(name='type0', target_type='Type0', doc='Group 0') + child1 = LinkSpec(name='type1', target_type='Type0', doc='Group 1') + child2 = LinkSpec(target_type='Type0', doc='Group 2') parent_spec = GroupSpec( - doc='A test group', + data_type_def='ParentType', name='parent', + doc='A test group', links=[child0, child1, child2], - data_type_def='ParentType' ) self.assertIs(parent_spec.get_target_type('Type0'), child2) def test_unnamed_named_link_same_type(self): """Test get_target_type when a group contains both an unnamed and named link with type X.""" - child0 = LinkSpec(doc='Group 0', target_type='Type0') - child1 = LinkSpec(doc='Group 1', target_type='Type0', name='type1') + child0 = LinkSpec(target_type='Type0', doc='Group 0') + child1 = LinkSpec(name='type1', target_type='Type0', doc='Group 1') parent_spec = GroupSpec( - doc='A test group', + data_type_def='ParentType', name='parent', + doc='A test group', links=[child0, child1], - data_type_def='ParentType' ) self.assertIs(parent_spec.get_target_type('Type0'), child0) def test_two_named_links_same_type(self): """Test get_target_type when a group contains multiple named links with type X.""" - child0 = LinkSpec(doc='Group 0', target_type='Type0', name='group0') - child1 = LinkSpec(doc='Group 1', target_type='Type0', name='group1') + child0 = LinkSpec(name='group0', target_type='Type0', doc='Group 0') + child1 = LinkSpec(name='group1', target_type='Type0', doc='Group 1') parent_spec = GroupSpec( - doc='A test group', + data_type_def='ParentType', name='parent', + doc='A test group', links=[child0, child1], - data_type_def='ParentType' ) self.assertEqual(parent_spec.get_target_type('Type0'), [child0, child1]) def test_three_named_links_same_type(self): """Test get_target_type when a group contains three named links with type X.""" - child0 = LinkSpec(doc='Group 0', target_type='Type0', name='type0') - child1 = LinkSpec(doc='Group 1', target_type='Type0', name='type1') - child2 = LinkSpec(doc='Group 2', target_type='Type0', name='type2') + child0 = LinkSpec(name='type0', target_type='Type0', doc='Group 0') + child1 = LinkSpec(name='type1', target_type='Type0', doc='Group 1') + child2 = LinkSpec(name='type2', target_type='Type0', doc='Group 2') parent_spec = GroupSpec( - doc='A test group', + data_type_def='ParentType', name='parent', + doc='A test group', links=[child0, child1, child2], - data_type_def='ParentType' ) self.assertEqual(parent_spec.get_target_type('Type0'), [child0, child1, child2]) @@ -538,27 +583,27 @@ def test_three_named_links_same_type(self): class SpecWithGroupsLinksTest(TestCase): def test_unnamed_group_link_same_type(self): - child = GroupSpec(doc='Group 0', data_type_inc='Type0') - link = LinkSpec(doc='Link 0', target_type='Type0') + child = GroupSpec(data_type_inc='Type0', doc='Group 0') + link = LinkSpec(target_type='Type0', doc='Link 0') parent_spec = GroupSpec( - doc='A test group', + data_type_def='ParentType', name='parent', + doc='A test group', groups=[child], links=[link], - data_type_def='ParentType' ) self.assertIs(parent_spec.get_data_type('Type0'), child) self.assertIs(parent_spec.get_target_type('Type0'), link) def test_unnamed_dataset_link_same_type(self): - child = DatasetSpec(doc='Dataset 0', data_type_inc='Type0') - link = LinkSpec(doc='Link 0', target_type='Type0') + child = DatasetSpec(data_type_inc='Type0', doc='Dataset 0') + link = LinkSpec(target_type='Type0', doc='Link 0') parent_spec = GroupSpec( - doc='A test group', + data_type_def='ParentType', name='parent', + doc='A test group', datasets=[child], links=[link], - data_type_def='ParentType' ) self.assertIs(parent_spec.get_data_type('Type0'), child) self.assertIs(parent_spec.get_target_type('Type0'), link) diff --git a/tests/unit/spec_tests/test_link_spec.py b/tests/unit/spec_tests/test_link_spec.py index 38e10886b..2bca909a7 100644 --- a/tests/unit/spec_tests/test_link_spec.py +++ b/tests/unit/spec_tests/test_link_spec.py @@ -1,6 +1,6 @@ -import json +from pydantic import ValidationError -from hdmf.spec import GroupSpec, LinkSpec +from hdmf.spec import LinkSpec, QuantityEnum from hdmf.testing import TestCase @@ -8,50 +8,37 @@ class LinkSpecTests(TestCase): def test_constructor(self): spec = LinkSpec( - doc='A test link', - target_type='Group1', - quantity='+', name='Link1', - ) - self.assertEqual(spec.doc, 'A test link') - self.assertEqual(spec.target_type, 'Group1') - self.assertEqual(spec.data_type_inc, 'Group1') - self.assertEqual(spec.quantity, '+') - self.assertEqual(spec.name, 'Link1') - json.dumps(spec) - - def test_constructor_target_spec_def(self): - group_spec_def = GroupSpec( - data_type_def='Group1', - doc='A test group', - ) - spec = LinkSpec( + target_type='Group1', doc='A test link', - target_type=group_spec_def, + quantity='?', ) + self.assertEqual(spec.name, 'Link1') + self.assertEqual(spec.doc, 'A test link') self.assertEqual(spec.target_type, 'Group1') - json.dumps(spec) - - def test_constructor_target_spec_inc(self): - group_spec_inc = GroupSpec( - data_type_inc='Group1', - doc='A test group', - ) - msg = "'target_type' must be a string or a GroupSpec or DatasetSpec with a 'data_type_def' key." - with self.assertRaisesWith(ValueError, msg): - LinkSpec( - doc='A test link', - target_type=group_spec_inc, - ) + self.assertEqual(spec.quantity, QuantityEnum.ZERO_OR_ONE) + spec_dict = spec.model_dump(exclude_unset=True) + expected = { + 'name': 'Link1', + 'doc': 'A test link', + 'target_type': 'Group1', + 'quantity': '?', + } + self.assertDictEqual(spec_dict, expected) def test_constructor_defaults(self): spec = LinkSpec( - doc='A test link', target_type='Group1', + doc='A test link', ) self.assertEqual(spec.quantity, 1) self.assertIsNone(spec.name) - json.dumps(spec) + model_dict = spec.model_dump(exclude_unset=True) + expected = { + "doc": "A test link", + "target_type": "Group1", + } + self.assertDictEqual(model_dict, expected) def test_required_is_many(self): quantity_opts = ['?', 1, '*', '+'] @@ -60,12 +47,11 @@ def test_required_is_many(self): for (quantity, req, many) in zip(quantity_opts, is_required, is_many): with self.subTest(quantity=quantity): spec = LinkSpec( - doc='A test link', target_type='Group1', + doc='A test link', quantity=quantity, - name='Link1', ) - self.assertEqual(spec.required, req) + self.assertEqual(spec.required, req) # TODO self.assertEqual(spec.is_many(), many) def test_build_warn_extra_args(self): @@ -75,7 +61,6 @@ def test_build_warn_extra_args(self): 'target_type': 'TestType', 'required': True, } - msg = ("Unexpected keys ['required'] in spec {'name': 'link1', 'doc': 'test link', " - "'target_type': 'TestType', 'required': True}") - with self.assertWarnsWith(UserWarning, msg): - LinkSpec.build_spec(spec_dict) + # TODO + with self.assertRaisesRegex(ValidationError, r"required\s*Extra inputs are not permitted"): + LinkSpec(**spec_dict) diff --git a/tests/unit/spec_tests/test_ref_spec.py b/tests/unit/spec_tests/test_ref_spec.py index 3277673d1..cb2c655d7 100644 --- a/tests/unit/spec_tests/test_ref_spec.py +++ b/tests/unit/spec_tests/test_ref_spec.py @@ -1,4 +1,4 @@ -import json +from pydantic import ValidationError from hdmf.spec import RefSpec from hdmf.testing import TestCase @@ -7,11 +7,16 @@ class RefSpecTests(TestCase): def test_constructor(self): - spec = RefSpec('TimeSeries', 'object') + spec = RefSpec(target_type='TimeSeries', reftype='object') self.assertEqual(spec.target_type, 'TimeSeries') self.assertEqual(spec.reftype, 'object') - json.dumps(spec) # to ensure there are no circular links + spec_dict = spec.model_dump(exclude_unset=True) + expected = { + 'target_type': 'TimeSeries', + 'reftype': 'object', + } + self.assertDictEqual(spec_dict, expected) def test_wrong_reference_type(self): - with self.assertRaises(ValueError): - RefSpec('TimeSeries', 'unknownreftype') + with self.assertRaisesRegex(ValidationError, r"reftype\s*Input should be 'object'"): + RefSpec(target_type='TimeSeries', reftype='unknownreftype') diff --git a/tests/unit/spec_tests/test_spec_resolution.py b/tests/unit/spec_tests/test_spec_resolution.py index d7c4a9d2a..01917ad8a 100644 --- a/tests/unit/spec_tests/test_spec_resolution.py +++ b/tests/unit/spec_tests/test_spec_resolution.py @@ -15,6 +15,7 @@ from hdmf.spec import ( AttributeSpec, + BaseStorageSpec, DatasetSpec, DtypeSpec, GroupSpec, @@ -23,7 +24,6 @@ SpecNamespace, NamespaceCatalog, ) -from hdmf.spec.spec import BaseStorageSpec from hdmf.testing import TestCase diff --git a/tests/unit/spec_tests/test_spec_write.py b/tests/unit/spec_tests/test_spec_write.py index 4bcebe8bf..071c392fa 100644 --- a/tests/unit/spec_tests/test_spec_write.py +++ b/tests/unit/spec_tests/test_spec_write.py @@ -2,7 +2,7 @@ import os from hdmf.spec.namespace import SpecNamespace, NamespaceCatalog -from hdmf.spec.spec import GroupSpec +from hdmf.spec import GroupSpec from hdmf.spec.write import NamespaceBuilder, YAMLSpecWriter, export_spec from hdmf.testing import TestCase diff --git a/tests/unit/test_io_hdf5_h5tools.py b/tests/unit/test_io_hdf5_h5tools.py index 5e8c5510d..c472feffa 100644 --- a/tests/unit/test_io_hdf5_h5tools.py +++ b/tests/unit/test_io_hdf5_h5tools.py @@ -24,7 +24,7 @@ from hdmf.data_utils import DataChunkIterator, GenericDataChunkIterator, InvalidDataIOError, append_data from hdmf.spec.catalog import SpecCatalog from hdmf.spec.namespace import NamespaceCatalog, SpecNamespace -from hdmf.spec.spec import GroupSpec, DtypeSpec +from hdmf.spec import GroupSpec, DtypeSpec from hdmf.testing import TestCase, remove_test_file from hdmf.common.resources import HERD from hdmf.term_set import TermSet, TermSetWrapper