feat(web-ui): add private-GitHub ingestion via PAT (#286)

* feat(web-ui, backend): allow ingesting private GitHub repos with PAT authentication

* Accept a GitHub personal access token (PAT) from the UI and forward it through
  - `git_form.jinja` → new “Private Repository” checkbox + PAT field
  - routers (`index.py`, `dynamic.py`) and `query_processor.py`
* Propagate `token` throughout the ingestion stack
  - `gitingest.entrypoint.parse_query`
  - `query_parsing` (including `try_domains_for_user_and_repo`) so we can infer the host when the user enters a bare “user/repo” slug
* Tests
  - Added `"token": ""` to the `form_data` dict in the tests in `tests/test_flow_integration.py`

**Limitation:** This PR enables PAT-protected cloning **only for GitHub**; other hosts (GitLab, Gitea, etc.) remain public-only for now.

* help link to generate PAT

* pre-commit hooks

---------

Co-authored-by: cyclotruc <romain@coderamp.io>

Author: filipchristiansen

src/gitingest/entrypoint.py

@@ -73,6 +73,7 @@ async def ingest_async(
             from_web=False,
             include_patterns=include_patterns,
             ignore_patterns=exclude_patterns,
+            token=token,
         )
 
         if query.url:

src/gitingest/query_parsing.py

@@ -29,6 +29,7 @@ async def parse_query(
     from_web: bool,
     include_patterns: Optional[Union[str, Set[str]]] = None,
     ignore_patterns: Optional[Union[str, Set[str]]] = None,
+    token: Optional[str] = None,
 ) -> IngestionQuery:
     """
     Parse the input source (URL or path) to extract relevant details for the query.
@@ -49,7 +50,10 @@ async def parse_query(
         Patterns to include, by default None. Can be a set of strings or a single string.
     ignore_patterns : Union[str, Set[str]], optional
         Patterns to ignore, by default None. Can be a set of strings or a single string.
-
+    token : str, optional
+        GitHub personal-access token (PAT). Needed when *source* refers to a
+        **private** repository. Can also be set via the ``GITHUB_TOKEN`` env var.
+        Must start with 'github_pat_' or 'gph_' for GitHub repositories.
     Returns
     -------
     IngestionQuery
@@ -59,7 +63,7 @@ async def parse_query(
     # Determine the parsing method based on the source type
     if from_web or urlparse(source).scheme in ("https", "http") or any(h in source for h in KNOWN_GIT_HOSTS):
         # We either have a full URL or a domain-less slug
-        query = await _parse_remote_repo(source)
+        query = await _parse_remote_repo(source, token=token)
     else:
         # Local path scenario
         query = _parse_local_dir_path(source)

src/server/query_processor.py

@@ -1,6 +1,7 @@
 """Process a query by parsing input, cloning a repository, and generating a summary."""
 
 from functools import partial
+from typing import Optional
 
 from fastapi import Request
 from starlette.templating import _TemplateResponse
@@ -19,6 +20,7 @@ async def process_query(
     pattern_type: str = "exclude",
     pattern: str = "",
     is_index: bool = False,
+    token: Optional[str] = None,
 ) -> _TemplateResponse:
     """
     Process a query by parsing input, cloning a repository, and generating a summary.
@@ -40,6 +42,9 @@ async def process_query(
         Pattern to include or exclude in the query, depending on the pattern type.
     is_index : bool
         Flag indicating whether the request is for the index page (default is False).
+    token : str, optional
+        GitHub personal-access token (PAT). Needed when *input_text* refers to a
+        **private** repository.
 
     Returns
     -------
@@ -71,6 +76,7 @@ async def process_query(
         "default_file_size": slider_position,
         "pattern_type": pattern_type,
         "pattern": pattern,
+        "token": token,
     }
 
     try:
@@ -80,12 +86,13 @@ async def process_query(
             from_web=True,
             include_patterns=include_patterns,
             ignore_patterns=exclude_patterns,
+            token=token,
         )
         if not query.url:
             raise ValueError("The 'url' parameter is required.")
 
         clone_config = query.extract_clone_config()
-        await clone_repo(clone_config)
+        await clone_repo(clone_config, token=token)
         summary, tree, content = ingest_query(query)
         with open(f"{clone_config.local_path}.txt", "w", encoding="utf-8") as f:
             f.write(tree + "\n" + content)

src/server/routers/dynamic.py

@@ -50,6 +50,7 @@ async def process_catch_all(
     max_file_size: int = Form(...),
     pattern_type: str = Form(...),
     pattern: str = Form(...),
+    token: str = Form(...),
 ) -> HTMLResponse:
     """
     Process the form submission with user input for query parameters.
@@ -69,18 +70,22 @@ async def process_catch_all(
         The type of pattern used for the query, specified by the user.
     pattern : str
         The pattern string used in the query, specified by the user.
-
+    token : str
+        GitHub personal-access token (PAT). Needed when *input_text* refers to a
+        **private** repository.
     Returns
     -------
     HTMLResponse
         An HTML response generated after processing the form input and query logic,
         which will be rendered and returned to the user.
     """
+    resolved_token = None if token == "" else token
     return await process_query(
         request,
         input_text,
         max_file_size,
         pattern_type,
         pattern,
         is_index=False,
+        token=resolved_token,
     )

src/server/routers/index.py

@@ -47,6 +47,7 @@ async def index_post(
     max_file_size: int = Form(...),
     pattern_type: str = Form(...),
     pattern: str = Form(...),
+    token: str = Form(...),
 ) -> HTMLResponse:
     """
     Process the form submission with user input for query parameters.
@@ -67,18 +68,22 @@ async def index_post(
         The type of pattern used for the query, specified by the user.
     pattern : str
         The pattern string used in the query, specified by the user.
-
+    token : str
+        GitHub personal-access token (PAT). Needed when *input_text* refers to a
+        **private** repository.
     Returns
     -------
     HTMLResponse
         An HTML response containing the results of processing the form input and query logic,
         which will be rendered and returned to the user.
     """
+    resolved_token = None if token == "" else token
     return await process_query(
         request,
         input_text,
         max_file_size,
         pattern_type,
         pattern,
         is_index=True,
+        token=resolved_token,
     )