refactor: centralize PAT validation, streamline repo checks & misc cleanup (#349)

* refactor: centralize PAT validation, streamline repo checks & housekeeping

* `.venv*` to `.gitignore`
* `# type: ignore[attr-defined]` hints in `compat_typing.py` for IDE-agnostic imports
* Helpful PAT string in `InvalidGitHubTokenError` for easier debugging

* Bump **ruff-pre-commit** hook → `v0.12.1`
* CONTRIBUTING:
  * Require **Python 3.9+**
  * Recommend signed (`-S`) commits
* PAT validation now happens **only** in entry points
  (`utils.auth.resolve_token` for CLI/lib, `server.process_query` for Web UI)
* Unified `_check_github_repo_exists` into `check_repo_exists`, replacing
  `curl -I` with `curl --silent --location --write-out %{http_code} -o /dev/null`
* Broaden `_GITHUB_PAT_PATTERN`
* `create_git_auth_header` raises `ValueError` when hostname is missing
* Tests updated to expect raw HTTP-code output

* Superfluous “token can be set via `GITHUB_TOKEN`” notes in docstrings
* `.gitingestignore` & `.terraform` from `DEFAULT_IGNORE_PATTERNS`
* Token validation inside `create_git_command`
* Obsolete `test_create_git_command_invalid_token`

* Adjust `test_clone.py` and `test_git_utils.py` for new status-code handling
* Consolidate mocks after token-validation relocation

BREAKING CHANGE:
`create_git_command` no longer validates GitHub tokens; callers must ensure
tokens are valid (via `validate_github_token`) before invoking lower-level
git helpers.

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: filipchristiansen

.pre-commit-config.yaml

@@ -151,4 +151,4 @@ repos:
   - repo: meta
     hooks:
       - id: check-hooks-apply
-      - id: check-useless-excludes
+      - id: check-useless-excludes

CONTRIBUTING.md

@@ -89,4 +89,4 @@ If you ever get stuck, reach out on [Discord](https://discord.com/invite/zerRaGK
 
 13. **Iterate** on any review feedback—update your branch and repeat **6 – 11** as needed.
 
-*(Optional) Invite a maintainer to your branch for easier collaboration.*
+*(Optional) Invite a maintainer to your branch for easier collaboration.*

src/gitingest/utils/git_utils.py

@@ -4,16 +4,28 @@
 
 import asyncio
 import base64
+import os
 import re
 from pathlib import Path
 from typing import TYPE_CHECKING, Final, Iterable
 import sys
+from typing import Final
 from urllib.parse import urlparse
 
 import httpx
 from starlette.status import HTTP_200_OK, HTTP_401_UNAUTHORIZED, HTTP_403_FORBIDDEN, HTTP_404_NOT_FOUND
 
 from gitingest.utils.compat_func import removesuffix
+
+from starlette.status import (
+    HTTP_200_OK,
+    HTTP_301_MOVED_PERMANENTLY,
+    HTTP_302_FOUND,
+    HTTP_401_UNAUTHORIZED,
+    HTTP_403_FORBIDDEN,
+    HTTP_404_NOT_FOUND,
+)
+
 from gitingest.utils.exceptions import InvalidGitHubTokenError
 from server.server_utils import Colors
 
@@ -134,28 +146,46 @@ async def check_repo_exists(url: str, token: str | None = None) -> bool:
         If the host returns an unrecognised status code.
 
     """
-    headers = {}
+    # TODO: use `requests` instead of `curl`
+    cmd: list[str] = [
+        "curl",
+        "--silent",
+        "--location",
+        "--head",
+        "--write-out",
+        "%{http_code}",
+        "-o",
+        os.devnull,
+    ]
 
     if token and is_github_host(url):
         host, owner, repo = _parse_github_url(url)
         # Public GitHub vs. GitHub Enterprise
         base_api = "https://api.github.com" if host == "github.com" else f"https://{host}/api/v3"
         url = f"{base_api}/repos/{owner}/{repo}"
-        headers["Authorization"] = f"Bearer {token}"
+        cmd += [f"Authorization: Bearer {token}"]
 
-    async with httpx.AsyncClient(follow_redirects=True) as client:
-        try:
-            response = await client.head(url, headers=headers)
-        except httpx.RequestError:
-            return False
+    cmd.append(url)
+
+    proc = await asyncio.create_subprocess_exec(
+        *cmd,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    stdout, _ = await proc.communicate()
 
-    status_code = response.status_code
+    if proc.returncode != 0:
+        return False
 
-    if status_code == HTTP_200_OK:
+    status = int(stdout.decode().strip())
+    if status in {HTTP_200_OK, HTTP_301_MOVED_PERMANENTLY}:
         return True
-    if status_code in {HTTP_401_UNAUTHORIZED, HTTP_403_FORBIDDEN, HTTP_404_NOT_FOUND}:
+    # TODO: handle 302 redirects
+    if status in {HTTP_404_NOT_FOUND, HTTP_302_FOUND}:
+        return False
+    if status in {HTTP_401_UNAUTHORIZED, HTTP_403_FORBIDDEN}:
         return False
-    msg = f"Unexpected HTTP status {status_code} for {url}"
+    msg = f"Unexpected HTTP status {status} for {url}"
     raise RuntimeError(msg)
 
 
@@ -187,7 +217,7 @@ def _parse_github_url(url: str) -> tuple[str, str, str]:
         msg = f"Un-recognised GitHub hostname: {parsed.hostname!r}"
         raise ValueError(msg)
 
-    parts = removesuffix(parsed.path, ".git").strip("/").split("/")
+    parts = parsed.path.strip("/").removesuffix(".git").split("/")
     expected_path_length = 2
     if len(parts) != expected_path_length:
         msg = f"Path must look like /<owner>/<repo>: {parsed.path!r}"
@@ -220,28 +250,13 @@ async def fetch_remote_branches_or_tags(url: str, *, ref_type: str, token: str |
         If the ``ref_type`` parameter is not "branches" or "tags".
 
     """
-    if ref_type not in ("branches", "tags"):
-        msg = f"Invalid fetch type: {ref_type}"
-        raise ValueError(msg)
-
     cmd = ["git"]
 
     # Add authentication if needed
     if token and is_github_host(url):
         cmd += ["-c", create_git_auth_header(token, url=url)]
 
-    cmd += ["ls-remote"]
-
-    fetch_tags = ref_type == "tags"
-    to_fetch = "tags" if fetch_tags else "heads"
-
-    cmd += [f"--{to_fetch}"]
-
-    # `--refs` filters out the peeled tag objects (those ending with "^{}") (for tags)
-    if fetch_tags:
-        cmd += ["--refs"]
-
-    cmd += [url]
+    cmd += ["ls-remote", "--heads", url]
 
     await ensure_git_installed()
     stdout, _ = await run_command(*cmd)
@@ -250,9 +265,9 @@ async def fetch_remote_branches_or_tags(url: str, *, ref_type: str, token: str |
     # - Skip empty lines and lines that don't contain "refs/{to_fetch}/"
     # - Extract the branch or tag name after "refs/{to_fetch}/"
     return [
-        line.split(f"refs/{to_fetch}/", 1)[1]
+        line.split("refs/heads/", 1)[1]
         for line in stdout.decode().splitlines()
-        if line.strip() and f"refs/{to_fetch}/" in line
+        if line.strip() and "refs/heads/" in line
     ]
 
 

src/server/query_processor.py

@@ -9,8 +9,12 @@
 from gitingest.ingestion import ingest_query
 from gitingest.query_parser import IngestionQuery, parse_query
 from gitingest.utils.git_utils import validate_github_token
-from server.models import IngestErrorResponse, IngestResponse, IngestSuccessResponse
-from server.server_config import MAX_DISPLAY_SIZE
+from server.server_config import (
+    DEFAULT_FILE_SIZE_KB,
+    EXAMPLE_REPOS,
+    MAX_DISPLAY_SIZE,
+    templates,
+)
 from server.server_utils import Colors, log_slider_to_size
 
 
@@ -63,6 +67,8 @@ async def process_query(
     if token:
         validate_github_token(token)
 
+    template = "index.jinja" if is_index else "git.jinja"
+    template_response = partial(templates.TemplateResponse, name=template)
     max_file_size = log_slider_to_size(slider_position)
 
     query: IngestionQuery | None = None
@@ -99,7 +105,10 @@ async def process_query(
             print(f"{Colors.BROWN}WARN{Colors.END}: {Colors.RED}<-  {Colors.END}", end="")
             print(f"{Colors.RED}{exc}{Colors.END}")
 
-        return IngestErrorResponse(error=str(exc), repo_url=short_repo_url)
+        context["error_message"] = f"Error: {exc}"
+        if "405" in str(exc):
+            context["error_message"] = "Repository not found. Please make sure it is public."
+        return template_response(context=context)
 
     if len(content) > MAX_DISPLAY_SIZE:
         content = (

tests/test_clone.py

@@ -91,10 +91,9 @@ async def test_clone_nonexistent_repository(repo_exists_true: AsyncMock) -> None
 @pytest.mark.parametrize(
     ("status_code", "expected"),
     [
-        (HTTP_200_OK, True),
-        (HTTP_401_UNAUTHORIZED, False),
-        (HTTP_403_FORBIDDEN, False),
-        (HTTP_404_NOT_FOUND, False),
+        (b"200\n", 0, True),  # Existing repo
+        (b"404\n", 0, False),  # Non-existing repo
+        (b"200\n", 1, False),  # Failed request
     ],
 )
 async def test_check_repo_exists(status_code: int, *, expected: bool, mocker: MockerFixture) -> None:
@@ -209,6 +208,25 @@ async def test_check_repo_exists_with_redirect(mocker: MockerFixture) -> None:
     assert repo_exists is False
 
 
+@pytest.mark.asyncio
+async def test_check_repo_exists_with_permanent_redirect(mocker: MockerFixture) -> None:
+    """Test ``check_repo_exists`` when a permanent redirect (301) is returned.
+
+    Given a URL that responds with "301 Found":
+    When ``check_repo_exists`` is called,
+    Then it should return ``True``, indicating the repo may exist at the new location.
+    """
+    mock_exec = mocker.patch("asyncio.create_subprocess_exec", new_callable=AsyncMock)
+    mock_process = AsyncMock()
+    mock_process.communicate.return_value = (b"301\n", b"")
+    mock_process.returncode = 0  # Simulate successful request
+    mock_exec.return_value = mock_process
+
+    repo_exists = await check_repo_exists(DEMO_URL)
+
+    assert repo_exists
+
+
 @pytest.mark.asyncio
 async def test_clone_with_timeout(run_command_mock: AsyncMock) -> None:
     """Test cloning a repository when a timeout occurs.