Remove RAG group (#1328)

* remove rag group

* add lock file

* remove chromadb dependent code

* remove chroma dependent tests

* remove entire rag group

* remove rag test

---------

Co-authored-by: TIANYOU CHEN <42710806+CTY-git@users.noreply.github.com>

Author: jonahdc

.github/workflows/test.yml

@@ -118,53 +118,6 @@ jobs:
           --force_pr_creation \
           --disable_telemetry
 
-  rag-test:
-    # disabled because this currently takes too long
-    if: false
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ github.head_ref }}
-
-      - uses: jwalton/gh-find-current-pr@master
-        id: findPr
-
-      - name: Set up python
-        id: setup-python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.9'
-
-      - name: Install Poetry
-        uses: snok/install-poetry@v1
-        with:
-          virtualenvs-create: true
-          virtualenvs-in-project: true
-          installer-parallel: true
-
-      - name: Load cached venv
-        id: cached-poetry-dependencies
-        uses: actions/cache@v4
-        with:
-          path: ./.venv
-          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('./poetry.lock') }}-${{ github.job }}
-
-      - name: Install dependencies
-        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction --only main --extras rag
-
-      - name: Resolve issue
-        run: |
-          source .venv/bin/activate
-          patchwork ResolveIssue --log debug \
-          --patched_api_key=${{ secrets.PATCHED_API_KEY }} \
-          --github_api_key=${{ secrets.SCM_GITHUB_KEY }} \
-          --issue_url=https://github.com/patched-codes/patchwork/issues/1129 \
-          --disable_telemetry
-          --max_llm_calls=10
-
   main-test:
     runs-on: ubuntu-latest
     steps:

patchwork/common/utils/dependency.py

@@ -2,7 +2,6 @@
 from functools import lru_cache
 
 __DEPENDENCY_GROUPS = {
-    "rag": ["chromadb"],
     "security": ["semgrep", "depscan"],
     "notification": ["slack_sdk"],
 }
@@ -22,9 +21,5 @@ def import_with_dependency_group(name):
         raise ImportError(error_msg)
 
 
-def chromadb():
-    return import_with_dependency_group("chromadb")
-
-
 def slack_sdk():
     return import_with_dependency_group("slack_sdk")

patchwork/common/utils/utils.py

@@ -15,9 +15,7 @@
 from git import Head, Repo
 from typing_extensions import Any, Callable, Counter
 
-from patchwork.common.utils.dependency import chromadb
 from patchwork.logger import logger
-from patchwork.managed_files import HOME_FOLDER
 
 _CLEANUP_FILES: set[Path] = set()
 _NEWLINES = {"\n", "\r\n", "\r"}
@@ -129,66 +127,6 @@ def count_openai_tokens(code: str):
     return len(_ENCODING.encode(code))
 
 
-def get_vector_db_path() -> str:
-    CHROMA_DB_PATH = HOME_FOLDER / "chroma.db"
-    if CHROMA_DB_PATH:
-        return str(CHROMA_DB_PATH)
-    else:
-        return ".chroma.db"
-
-
-def openai_embedding_model(
-    inputs: dict,
-) -> "chromadb.api.types.EmbeddingFunction"["chromadb.api.types.Documents"] | None:
-    model = inputs.get(openai_embedding_model.__name__)
-    if model is None:
-        return None
-
-    api_key = inputs.get("openai_api_key")
-    if api_key is None:
-        raise ValueError("Missing required input data: 'openai_api_key'")
-
-    return chromadb().utils.embedding_functions.OpenAIEmbeddingFunction(
-        api_key=api_key,
-        model_name=model,
-    )
-
-
-def huggingface_embedding_model(
-    inputs: dict,
-) -> "chromadb.api.types.EmbeddingFunction"["chromadb.api.types.Documents"] | None:
-    model = inputs.get(huggingface_embedding_model.__name__)
-    if model is None:
-        return None
-
-    api_key = inputs.get("openai_api_key") or inputs.get("huggingface_api_key")
-    if api_key is None:
-        raise ValueError("Missing required input data: 'openai_api_key' or 'huggingface_api_key'")
-
-    return chromadb().utils.embedding_functions.HuggingFaceEmbeddingFunction(
-        api_key=api_key,
-        model_name=model,
-    )
-
-
-_EMBEDDING_FUNCS = [openai_embedding_model, huggingface_embedding_model]
-
-_EMBEDDING_TO_API_KEY_NAME: dict[
-    str, Callable[[dict], "chromadb.api.type.EmbeddingFunction"["chromadb.api.types.Documents"] | None]
-] = {func.__name__: func for func in _EMBEDDING_FUNCS}
-
-
-def get_embedding_function(inputs: dict) -> "chromadb.api.types.EmbeddingFunction"["chromadb.api.types.Documents"]:
-    embedding_function = next(
-        (func(inputs) for input_key, func in _EMBEDDING_TO_API_KEY_NAME.items() if input_key in inputs.keys()),
-        chromadb().utils.embedding_functions.SentenceTransformerEmbeddingFunction(),
-    )
-    if embedding_function is None:
-        raise ValueError(f"Missing required input data: one of {_EMBEDDING_TO_API_KEY_NAME.keys()}")
-
-    return embedding_function
-
-
 def get_current_branch(repo: Repo) -> Head:
     remote = repo.remote("origin")
     if repo.head.is_detached: