wip

Author: NicolasIRAGNE

src/gitingest/ingestion.py

@@ -6,9 +6,9 @@
 from typing import TYPE_CHECKING
 
 from gitingest.config import MAX_DIRECTORY_DEPTH, MAX_FILES, MAX_TOTAL_SIZE_BYTES
-from gitingest.output_formatter import DefaultFormatter, StupidFormatter
+from gitingest.output_formatter import DefaultFormatter, DebugFormatter, SummaryFormatter
 from gitingest.schemas import FileSystemNode, FileSystemStats, Context
-from gitingest.schemas.filesystem import FileSystemDirectory, FileSystemFile, FileSystemSymlink, FileSystemTextFile
+from gitingest.schemas.filesystem import FileSystemDirectory, FileSystemFile, FileSystemSymlink
 from gitingest.utils.ingestion_utils import _should_exclude, _should_include
 from gitingest.utils.logging_config import get_logger
 
@@ -70,14 +70,6 @@ def ingest_query(query: IngestionQuery) -> Context:
 
         relative_path = path.relative_to(query.local_path)
 
-        # file_node = FileSystemNode(
-        #     name=path.name,
-        #     type=FileSystemNodeType.FILE,
-        #     size=path.stat().st_size,
-        #     file_count=1,
-        #     path_str=str(relative_path),
-        #     path=path,
-        # )
         file_node = FileSystemFile(
             name=path.name,
             path_str=str(relative_path),
@@ -96,7 +88,7 @@ def ingest_query(query: IngestionQuery) -> Context:
                 "file_size": file_node.size,
             },
         )
-        return Context([file_node], StupidFormatter(), query)
+        return Context([file_node], query)
 
     # root_node = FileSystemNode(
     #     name=path.name,
@@ -125,7 +117,7 @@ def ingest_query(query: IngestionQuery) -> Context:
         },
     )
 
-    return Context([root_node], StupidFormatter(), query)
+    return Context([root_node], query)
 
 
 def _process_node(node: FileSystemNode, query: IngestionQuery, stats: FileSystemStats) -> None:
@@ -264,22 +256,13 @@ def _process_file(path: Path, parent_node: FileSystemDirectory, stats: FileSyste
     stats.total_files += 1
     stats.total_size += file_size
 
-    # if file is a .txt file, create a FileSystemTextFile
-    if path.suffix == ".txt":
-        child = FileSystemTextFile(
-            name=path.name,
-            path_str=str(path.relative_to(local_path)),
-            path=path,
-            depth=parent_node.depth + 1,
-        )
-    else:
 
-        child = FileSystemFile(
-            name=path.name,
-            path_str=str(path.relative_to(local_path)),
-            path=path,
-            depth=parent_node.depth + 1,
-        )
+    child = FileSystemFile(
+        name=path.name,
+        path_str=str(path.relative_to(local_path)),
+        path=path,
+        depth=parent_node.depth + 1,
+    )
 
     parent_node.children.append(child)
     parent_node.size += file_size
@@ -290,7 +273,7 @@ def limit_exceeded(stats: FileSystemStats, depth: int) -> bool:
     """Check if any of the traversal limits have been exceeded.
 
     This function checks if the current traversal has exceeded any of the configured limits:
-    maximum directory depth, maximum number of files, or maximum total size in bytes.
+    maximum directory depth, ma ximum number of files, or maximum total size in bytes.
 
     Parameters
     ----------

src/gitingest/output_formatter.py

@@ -11,36 +11,11 @@
 from gitingest.schemas import FileSystemNode
 from gitingest.utils.compat_func import readlink
 from functools import singledispatchmethod
-from gitingest.schemas import Source, FileSystemFile, FileSystemDirectory, FileSystemSymlink, FileSystemTextFile
-from gitingest.schemas.filesystem import SEPARATOR, FileSystemNodeType, CONTEXT_HEADER, CONTEXT_FOOTER
+from gitingest.schemas import Source, FileSystemFile, FileSystemDirectory, FileSystemSymlink
+from gitingest.schemas.filesystem import SEPARATOR, Context, FileSystemNodeType
 from gitingest.utils.logging_config import get_logger
 from jinja2 import Environment, BaseLoader
 
-
-class OverridableDispatcher:
-    """Custom dispatcher that allows later registrations to override earlier ones, even for parent types."""
-
-    def __init__(self, default_func):
-        self.default_func = default_func
-        self.registry = []  # List of (type, func) in registration order
-
-    def register(self, type_):
-        def decorator(func):
-            # Remove any existing registration for this exact type
-            self.registry = [(t, f) for t, f in self.registry if t != type_]
-            # Add new registration at the end (highest priority)
-            self.registry.append((type_, func))
-            return func
-        return decorator
-
-    def __call__(self, instance, *args, **kwargs):
-        # Check registrations in reverse order (most recent first)
-        for type_, func in reversed(self.registry):
-            if isinstance(instance, type_):
-                return func(instance, *args, **kwargs)
-        # Fall back to default
-        return self.default_func(instance, *args, **kwargs)
-
 if TYPE_CHECKING:
     from gitingest.schemas import IngestionQuery
 
@@ -202,116 +177,164 @@ def _format_token_count(text: str) -> str | None:
 
     return str(total_tokens)
 
+
+def generate_digest(context: Context) -> str:
+    """Generate a digest string from a Context object.
+    
+    This is a convenience function that uses the DefaultFormatter to format a Context.
+    
+    Parameters
+    ----------
+    context : Context
+        The Context object containing sources and query information.
+    
+    Returns
+    -------
+    str
+        The formatted digest string.
+    """
+    formatter = DefaultFormatter()
+    return formatter.format(context, context.query)
+
+
 class DefaultFormatter:
     def __init__(self):
         self.separator = SEPARATOR
         self.env = Environment(loader=BaseLoader())
 
-        # Set up custom dispatchers
-        def _default_format(node: Source, query):
-            return f"{getattr(node, 'content', '')}"
-
-        def _default_summary(node: Source, query):
-            return f"{getattr(node, 'name', '')}"
+    @singledispatchmethod
+    def format(self, node: Source, query):
+        return f"{getattr(node, 'content', '')}"
 
-        self.format = OverridableDispatcher(_default_format)
-        self.summary = OverridableDispatcher(_default_summary)
-
-        # Register the default implementations
-        self._register_defaults()
-
-    def _register_defaults(self):
-        @self.format.register(FileSystemFile)
-        def _(node: FileSystemFile, query):
-            template = \
+    @format.register
+    def _(self, node: FileSystemFile, query):
+        template = \
 """
 {{ SEPARATOR }}
 {{ node.name }}
 {{ SEPARATOR }}
 
 {{ node.content }}
 """
-            file_template = self.env.from_string(template)
-            return file_template.render(SEPARATOR=SEPARATOR, node=node, query=query, formatter=self)
+        file_template = self.env.from_string(template)
+        return file_template.render(SEPARATOR=SEPARATOR, node=node, query=query, formatter=self)
 
-        @self.format.register(FileSystemDirectory)
-        def _(node: FileSystemDirectory, query):
-            template = \
+    @format.register
+    def _(self, node: FileSystemDirectory, query):
+        template = \
 """
+{% if node.depth == 0 %}
+{{ node.name }}:
+{{ node.tree }}
+
+{% endif %}
 {% for child in node.children %}
 {{ formatter.format(child, query) }}
 {% endfor %}
 """
-            dir_template = self.env.from_string(template)
-            return dir_template.render(node=node, query=query, formatter=self)
+        dir_template = self.env.from_string(template)
+        return dir_template.render(node=node, query=query, formatter=self)
 
-        @self.summary.register(FileSystemDirectory)
-        def _(node: FileSystemDirectory, query):
-            template = \
-"""
-Directory structure:
-{{ node.tree }}
-"""
-            summary_template = self.env.from_string(template)
-            return summary_template.render(node=node, query=query, formatter=self)
-
-        @self.format.register(FileSystemSymlink)
-        def _(node: FileSystemSymlink, query):
-            template = \
+    @format.register
+    def _(self, node: FileSystemSymlink, query):
+        template = \
 """
 {{ SEPARATOR }}
 {{ node.name }}{% if node.target %} -> {{ node.target }}{% endif %}
 {{ SEPARATOR }}
 """
-            symlink_template = self.env.from_string(template)
-            return symlink_template.render(SEPARATOR=SEPARATOR, node=node, query=query, formatter=self)
+        symlink_template = self.env.from_string(template)
+        return symlink_template.render(SEPARATOR=SEPARATOR, node=node, query=query, formatter=self)
+
+    @format.register
+    def _(self, context: Context, query):
+        """Format a Context by formatting all its sources."""
+        template = \
+"""
+# Generated using https://gitingest.com/{{ context.query.user_name }}/{{ context.query.repo_name }}
+Sources used:
+{% for source in context.sources %}
+- {{ source.name }}: {{ source.__class__.__name__ }}
+{% endfor %}
+
+{% for source in context.sources %}
+{{ formatter.format(source, context.query) }}
+{% endfor %}
+# End of generated content
+"""
+        context_template = self.env.from_string(template)
+        return context_template.render(context=context, formatter=self)
 
-class StupidFormatter(DefaultFormatter):
+
+class DebugFormatter:
     def __init__(self):
-        super().__init__()
+        self.separator = SEPARATOR
+        self.env = Environment(loader=BaseLoader())
 
-        @self.summary.register(FileSystemTextFile)
-        def _(node: FileSystemTextFile, query):
-            template = \
+    @singledispatchmethod
+    def format(self, node: Source, query):
+        """Format any Source type with debug information."""
+        # Get the actual class name
+        class_name = node.__class__.__name__
+
+        # Get all field names (both from dataclass fields and regular attributes)
+        field_names = []
+
+        # Try to get dataclass fields first
+        try:
+            if hasattr(node, '__dataclass_fields__') and hasattr(node.__dataclass_fields__, 'keys'):
+                field_names.extend(node.__dataclass_fields__.keys())
+            else:
+                raise AttributeError  # Fall through to backup method
+        except (AttributeError, TypeError):
+            # Fall back to getting all non-private attributes
+            field_names = [attr for attr in dir(node)
+                          if not attr.startswith('_')
+                          and not callable(getattr(node, attr, None))]
+
+        # Format the debug output
+        fields_str = ", ".join(field_names)
+        template = \
 """
 {{ SEPARATOR }}
-{{ node.name }}
+DEBUG: {{ class_name }}
+Fields: {{ fields_str }}
 {{ SEPARATOR }}
-FileSystemTextFile
 """
+        debug_template = self.env.from_string(template)
+        return debug_template.render(
+            SEPARATOR=SEPARATOR,
+            class_name=class_name,
+            fields_str=fields_str
+        )
 
-        @self.format.register(FileSystemFile)
-        def _(node: FileSystemFile, query):
-            template = \
+
+class SummaryFormatter:
+    """Dedicated formatter for generating summaries of filesystem nodes."""
+
+    def __init__(self):
+        self.env = Environment(loader=BaseLoader())
+
+    @singledispatchmethod
+    def summary(self, node: Source, query):
+        return f"{getattr(node, 'name', '')}"
+
+    @summary.register
+    def _(self, node: FileSystemDirectory, query):
+        template = \
 """
-{{ SEPARATOR }}
-{{ node.name }}
-{{ SEPARATOR }}
-FileSystemFile
+Directory structure:
+{{ node.tree }}
 """
-            file_template = self.env.from_string(template)
-            return file_template.render(SEPARATOR=SEPARATOR, node=node, query=query, formatter=self)
+        summary_template = self.env.from_string(template)
+        return summary_template.render(node=node, query=query)
 
 
-def generate_digest(context) -> str:
-    """Generate a digest from a Context object.
-    
-    Parameters
-    ----------
-    context : Context
-        The context object containing nodes, formatter, and query.
-        
-    Returns
-    -------
-    str
-        The formatted digest string with header, content, and footer.
-    """
-    if context.query.user_name and context.query.repo_name:
-        context_header = CONTEXT_HEADER.format(f"/{context.query.user_name}/{context.query.repo_name}")
-    else:
-        context_header = CONTEXT_HEADER.format("")
-    context_footer = CONTEXT_FOOTER
-    formatted = []
-    for node in context.nodes:
-        formatted.append(context.formatter.format(node, context.query))
-    return context_header + "\n".join(formatted) + context_footer
+    @summary.register
+    def _(self, context: Context, query):
+        template = \
+"""
+{{ context.summary }}
+"""
+        summary_template = self.env.from_string(template)
+        return summary_template.render(context=context, query=query)

src/gitingest/schemas/__init__.py

@@ -1,7 +1,7 @@
 """Module containing the schemas for the Gitingest package."""
 
 from gitingest.schemas.cloning import CloneConfig
-from gitingest.schemas.filesystem import FileSystemNode, FileSystemFile, FileSystemDirectory, FileSystemSymlink, FileSystemTextFile, FileSystemStats, Context, Source
+from gitingest.schemas.filesystem import FileSystemNode, FileSystemFile, FileSystemDirectory, FileSystemSymlink, FileSystemStats, Context, Source
 from gitingest.schemas.ingestion import IngestionQuery
 
-__all__ = ["CloneConfig", "FileSystemNode", "FileSystemFile", "FileSystemDirectory", "FileSystemSymlink", "FileSystemTextFile", "FileSystemStats", "IngestionQuery", "Context"]
+__all__ = ["CloneConfig", "FileSystemNode", "FileSystemFile", "FileSystemDirectory", "FileSystemSymlink", "FileSystemStats", "IngestionQuery", "Context"]