🚧 Render all dynamic modules, including empty __init__ from a single template.

Author: rafalkrupinski

.pre-commit-config.yaml

@@ -8,6 +8,7 @@ repos:
   hooks:
   - id: trailing-whitespace
   - id: end-of-file-fixer
+    exclude: ^src/lapidary/render/templates/render/includes/header.txt$|^tests/e2e/expected/
   - id: check-added-large-files
   - id: check-toml
   - id: debug-statements

src/lapidary/render/main.py

@@ -113,7 +113,7 @@ async def render_project(project_root: anyio.Path) -> None:
                 model=model,
                 get_version=importlib.metadata.version,
             ),
-            pathlib.Path(project_root),
+            pathlib.Path(project_root) / 'gen',
             event_sink=event_sink,
             remove_stale=True,
         )
@@ -122,8 +122,8 @@ async def render_project(project_root: anyio.Path) -> None:
 class RenderProgressBar(rybak.EventSink):
     def __init__(self, model: python.ClientModel) -> None:
         self._progress_bar = click.progressbar(
-            model.schemas,
-            label='Rendering schemas',
+            model.modules,
+            label='Rendering modules',
             item_show_func=lambda item: item or '',
             show_pos=True,
         )
@@ -136,8 +136,9 @@ def __exit__(self, exc_type, exc_val, exc_tb) -> bool | None:
         return self._progress_bar.__exit__(exc_type, exc_val, exc_tb)
 
     def writing_file(self, template: pathlib.PurePath, target: pathlib.Path) -> None:
-        if str(template) == 'gen/{{loop_over(model.schemas).path.to_path()}}.jinja':
-            self._progress_bar.update(1, str(target).split('/')[-3])
+        super().writing_file(template, target)
+        if str(template) == '{{model.package}}/{{loop_over(model.modules).rel_path}}.py.jinja':
+            self._progress_bar.update(1, str(target).split('/')[-2])
 
 
 async def dump_model(project_root: anyio.Path, process: bool, output: TextIO):

src/lapidary/render/model/python/__init__.py

@@ -31,7 +31,8 @@
 )
 
 import dataclasses as dc
-from collections.abc import Iterable, MutableMapping, MutableSequence
+from collections.abc import Iterable, MutableMapping, MutableSequence, MutableSet, Sequence
+from functools import cached_property
 from typing import Self
 
 from ..openapi import ParameterLocation as ParamLocation
@@ -56,7 +57,7 @@
     SchemaClass,
     SecurityRequirements,
 )
-from .module import AuthModule, ClientModule, SchemaModule
+from .module import AbstractModule, AuthModule, ClientModule, EmptyModule, SchemaModule
 from .module_path import ModulePath
 from .type_hint import NONE, BuiltinTypeHint, GenericTypeHint, TypeHint, type_hint_or_union
 
@@ -71,13 +72,22 @@ class ClientModel:
     def packages(self: Self) -> Iterable[ModulePath]:
         # Used to create __init__.py files in otherwise empty packages
 
-        packages = {ModulePath(self.package)}
+        known_packages: MutableSet[ModulePath] = {ModulePath.root()}
 
         # for each schema module get its package
         for schema in self.schemas:
-            path = schema.path
-            while path_ := path.parent():
-                packages.add(path_)
-                path = path_
+            path: ModulePath | None = schema.path
+            while path := path.parent():
+                if path in known_packages:
+                    continue
+                yield path
+                known_packages.add(path)
 
-        return packages
+    @cached_property
+    def modules(self) -> Sequence[AbstractModule]:
+        return list(self._modules())
+
+    def _modules(self) -> Iterable[AbstractModule]:
+        yield from self.schemas
+        for package in self.packages():
+            yield EmptyModule(path=package, body=None)

src/lapidary/render/model/python/module.py

@@ -14,24 +14,29 @@
 
 
 @dc.dataclass(frozen=True, kw_only=True)
-class AbstractModule(abc.ABC):
-    path: ModulePath
+class AbstractModule[Body](abc.ABC):
+    path: ModulePath = dc.field()
+    module_type: str
+    body: Body = dc.field()
 
     @property
     @abc.abstractmethod
     def imports(self) -> Iterable[str]:
         pass
 
+    @property
+    def rel_path(self) -> str:
+        return self.path.to_path()
+
 
 @dc.dataclass(frozen=True, kw_only=True)
-class SchemaModule(AbstractModule):
+class SchemaModule(AbstractModule[Iterable[SchemaClass]]):
     """
     One schema module per schema element directly under #/components/schemas, containing that schema and all non-reference schemas.
     One schema module for inline request and for response body for each operation
     """
 
-    body: list[SchemaClass] = dc.field(default_factory=list)
-    model_type: str = 'schema'
+    module_type: str = 'schema'
 
     @property
     def imports(self) -> Iterable[str]:
@@ -47,19 +52,33 @@ def imports(self) -> Iterable[str]:
 
 
 @dc.dataclass(frozen=True, kw_only=True)
-class AuthModule(AbstractModule):
-    schemes: Mapping[str, TypeHint] = dc.field()
-    model_type = 'auth'
+class AuthModule(AbstractModule[Mapping[str, TypeHint]]):
+    module_type = 'auth'
+
+    @property
+    def imports(self) -> Iterable[str]:
+        yield from ()
 
 
 @dc.dataclass(frozen=True, kw_only=True)
-class ClientModule(AbstractModule):
-    body: ClientClass = dc.field()
-    model_type = 'client'
+class ClientModule(AbstractModule[ClientClass]):
+    module_type: str = dc.field(default='client')
 
     @property
     def imports(self) -> Iterable[str]:
         dependencies = GenericTypeHint.union_of(*self.body.dependencies).args  # flatten unions
         imports = sorted({imp for dep in dependencies if dep for imp in dep.imports() if imp not in template_imports})
 
         return imports
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class EmptyModule(AbstractModule[None]):
+    """Module used to generate empty __init__.py files"""
+
+    module_type: str = dc.field(default='empty')
+    body: None = None
+
+    @property
+    def imports(self) -> Iterable[str]:
+        yield from ()

src/lapidary/render/model/python/module_path.py

@@ -1,16 +1,17 @@
 import typing
 from collections.abc import Iterable
 from pathlib import PurePath
+from typing import Self
 
 
 class ModulePath:
     _SEP = '.'
 
-    def __init__(self, module: str | Iterable[str]):
+    def __init__(self, module: str | Iterable[str], is_module: bool = True):
         if isinstance(module, str):
             module = module.strip()
-            if module == '' or module.strip() != module:
-                raise ValueError()
+            if module.strip() != module:
+                raise ValueError(f'"{module}"')
             parts: Iterable[str] = module.split(ModulePath._SEP)
         else:
             parts = module
@@ -22,19 +23,22 @@ def __init__(self, module: str | Iterable[str]):
         else:
             raise ValueError(module)
 
-    def to_path(self, root: PurePath | None = None, is_module=True):
-        path = (root or PurePath()).joinpath(*self.parts)
-        if is_module:
-            name = self.parts[-1]
-            dot_idx = name.rfind('.')
-            suffix = name[dot_idx:] if dot_idx != -1 else '.py'
-            path = path.with_suffix(suffix)
-        return path
+        self._is_module = is_module
+
+    def to_path(self, root: PurePath | None = None):
+        parts = list(self.parts)
+        if not self._is_module:
+            parts.append('__init__')
+        return (root or PurePath()).joinpath(*parts)
 
     def parent(self) -> typing.Self | None:
         if len(self.parts) == 1:
             return None
-        return ModulePath(self.parts[:-1])
+        return ModulePath(self.parts[:-1], False)
+
+    @classmethod
+    def root(cls) -> Self:
+        return cls('', is_module=False)
 
     def __truediv__(self, other: str | Iterable[str]):
         if isinstance(other, str):
@@ -51,3 +55,12 @@ def __eq__(self, other: object):
 
     def __hash__(self) -> int:
         return hash(self.__str__())
+
+    def __matmul__(self, other):
+        if not isinstance(other, ModulePath):
+            return NotImplemented
+
+        if self.parts[: len(other.parts)] != other.parts:
+            raise ValueError('Not related')
+
+        return ModulePath(self.parts[len(other.parts) :])

src/lapidary/render/model/python/type_hint.py

@@ -118,7 +118,7 @@ def __hash__(self) -> int:
 
 class NoneTypeHint(TypeHint):
     def __init__(self):
-        pass
+        super().__init__(module=None, name=None)
 
     def full_name(self):
         return 'None'

src/lapidary/render/model/schema.py

@@ -141,10 +141,10 @@ def schema_modules(self) -> Iterable[python.SchemaModule]:
         modules: dict[python.ModulePath, list[python.SchemaClass]] = defaultdict(list)
         for pointer, schema_class_type in self.schema_types.items():
             schema_class, hint = schema_class_type
-            modules[python.ModulePath(hint.module) / '__init__'].append(schema_class)
+            modules[python.ModulePath(hint.module, True)].append(schema_class)
         return [
             python.SchemaModule(
-                path=module,
+                path=module @ self.root_package,
                 body=classes,
             )
             for module, classes in modules.items()

…).to_path(is_module=False)}}/__init__.py …es/render/includes/module/empty.py.jinjasrc/lapidary/render/templates/render/gen/{{loop_over(model.packages()).to_path(is_module=False)}}/__init__.py renamed to src/lapidary/render/templates/render/includes/module/empty.py.jinja src/lapidary/render/templates/render/gen/{{loop_over(model.packages()).to_path(is_module=False)}}/__init__.py renamed to src/lapidary/render/templates/render/includes/module/empty.py.jinja

@@ -1,4 +1,3 @@
-# {% include 'includes/header.txt' %}
 from __future__ import annotations
 
 import typing

…er(model.schemas).path.to_path()}}.jinja …s/render/includes/module/schema.py.jinjasrc/lapidary/render/templates/render/gen/{{loop_over(model.schemas).path.to_path()}}.jinja renamed to src/lapidary/render/templates/render/includes/module/schema.py.jinja src/lapidary/render/templates/render/gen/{{loop_over(model.schemas).path.to_path()}}.jinja renamed to src/lapidary/render/templates/render/includes/module/schema.py.jinja

@@ -0,0 +1,3 @@
+# {% include 'includes/header.txt' %}
+
+from .client import ApiClient