Sync chromasql from 4abf1948cdb8ced8924e7450f0d64fbb17592454

Author: darshitac11

CLI_README.md

@@ -235,7 +235,7 @@ The CLI provides a similar developer experience to the factory pattern in `adri_
 
 ### Import Errors
 
-If you see import errors related to `indexer` or `adri_agents`:
+If you see import errors related to `idxr` or `adri_agents`:
 
 - Make sure you're running the CLI from the main project: `poetry run chromasql-server`
 - Ensure all dependencies are installed: `poetry install`

CONTRIBUTING.md

@@ -145,9 +145,9 @@ clause. ChromaSQL provides a generic `MetadataFieldRouter` adapter:
 from pathlib import Path
 from chromasql.adapters import MetadataFieldRouter
 from chromasql.multi_collection import execute_multi_collection
-from indexer.query_lib.async_multi_collection_adapter import AsyncMultiCollectionAdapter
-from indexer.vectorize_lib.query_client import AsyncMultiCollectionQueryClient
-from indexer.vectorize_lib.query_config import load_query_config
+from idxr.query_lib.async_multi_collection_adapter import AsyncMultiCollectionAdapter
+from idxr.vectorize_lib.query_client import AsyncMultiCollectionQueryClient
+from idxr.vectorize_lib.query_config import load_query_config
 
 # Load your query config (maps discriminator values to collections)
 config = load_query_config(Path("query_config.json"))

_ast_nodes.py

@@ -32,6 +32,7 @@
     "BetweenPredicate",
     "LikePredicate",
     "ContainsPredicate",
+    "RegexPredicate",
     "BooleanPredicate",
     "Query",
 ]
@@ -136,12 +137,21 @@ class BetweenPredicate(Predicate):
 class LikePredicate(Predicate):
     field: Field
     pattern: str
+    negated: bool = False
 
 
 @dataclass(frozen=True)
 class ContainsPredicate(Predicate):
     field: Field
     value: Union[str, int, float, bool]
+    negated: bool = False
+
+
+@dataclass(frozen=True)
+class RegexPredicate(Predicate):
+    field: Field
+    pattern: str
+    negated: bool = False
 
 
 @dataclass(frozen=True)

adapters.py

@@ -11,8 +11,8 @@
         MetadataFieldRouter,
     )
     from chromasql.multi_collection import execute_multi_collection
-    from indexer.vectorize_lib.query_client import AsyncMultiCollectionQueryClient
-    from indexer.query_lib.async_multi_collection_adapter import AsyncMultiCollectionAdapter
+    from idxr.vectorize_lib.query_client import AsyncMultiCollectionQueryClient
+    from idxr.query_lib.async_multi_collection_adapter import AsyncMultiCollectionAdapter
 
     # Setup your existing client
     client = AsyncMultiCollectionQueryClient(

cli.py

@@ -136,7 +136,7 @@ def create_collection_environment_from_config(
     Raises:
         ValueError: If configuration is invalid or missing required fields
     """
-    from indexer.models import CollectionEnvironment
+    from idxr.models import CollectionEnvironment
 
     client_type = config.get("type", "").lower()
 
@@ -175,12 +175,13 @@ def create_collection_environment_from_config(
                 module_path = ".".join(rel_path.parts) + ".test_registry:MODEL_REGISTRY"
                 model_registry_target = module_path
             else:
-                model_registry_target = "indexer.registry:MODEL_REGISTRY"
+                model_registry_target = "idxr.registry:MODEL_REGISTRY"
 
         embedding_model = config.get("embedding_model", "text-embedding-3-small")
         local_collection_name = config.get("collection_name", collection_name)
 
         return CollectionEnvironment(
+            collection_name=local_collection_name,
             query_config_path=query_config_path,
             discriminator_field=discriminator_field,
             model_registry_target=model_registry_target,
@@ -214,11 +215,18 @@ def create_collection_environment_from_config(
 
         discriminator_field = config.get("discriminator_field", "model_name")
         model_registry_target = config.get(
-            "model_registry_target", "indexer.registry:MODEL_REGISTRY"
+            "model_registry_target", "idxr.registry:MODEL_REGISTRY"
         )
         embedding_model = config.get("embedding_model", "text-embedding-3-small")
 
+        # For cloud collections, use database name as collection_name or
+        # fallback to provided name
+        cloud_collection_name = config.get(
+            "collection_name", database or collection_name
+        )
+
         return CollectionEnvironment(
+            collection_name=cloud_collection_name,
             query_config_path=query_config_path,
             discriminator_field=discriminator_field,
             model_registry_target=model_registry_target,

collection_service.py

@@ -12,7 +12,7 @@
 from fastapi import HTTPException
 from pydantic import BaseModel
 
-from indexer.models import CollectionEnvironment as ResearchAgentEnvironment
+from idxr.models import CollectionEnvironment
 
 try:
     from adri_agents.app.utils.uni_vectordb_agent_harness.query_executor_config import (
@@ -21,11 +21,11 @@
     )
 except ImportError:
     # Fallback for standalone chromasql usage
-    from indexer.query_lib.config import (  # type: ignore[no-redef]
+    from idxr.query_lib.config import (  # type: ignore[no-redef]
         build_query_executor_kwargs,
         load_model_registry,
     )
-from indexer.query_lib.executor import QueryExecutor
+from idxr.query_lib.executor import QueryExecutor
 from chromasql.errors import (
     ChromaSQLParseError,
     ChromaSQLPlanningError,
@@ -35,7 +35,7 @@
 logger = getLogger(__name__)
 
 
-# System metadata fields automatically managed by the indexer
+# System metadata fields automatically managed by the idxr
 SYSTEM_METADATA_FIELDS: tuple[tuple[str, str, str], ...] = (
     ("model_name", "string", "String"),
     ("source_path", "string", "String"),
@@ -82,7 +82,7 @@ class CollectionService:
     1. Retrieving index metadata (for /indices endpoint)
     2. Executing ChromaSQL queries (for /chromasql/execute endpoint)
 
-    It can work with any ResearchAgentEnvironment, supporting both
+    It can work with any CollectionEnvironment, supporting both
     system collections and user-created collections.
 
     Usage:
@@ -107,7 +107,7 @@ def __init__(
         self,
         collection_name: str,
         display_name: str,
-        system_env: ResearchAgentEnvironment,
+        system_env: CollectionEnvironment,
     ):
         """
         Initialize the collection service for a single collection.
@@ -123,13 +123,13 @@ def __init__(
 
     @classmethod
     def from_env_map(
-        cls, env_map: Dict[str, ResearchAgentEnvironment]
+        cls, env_map: Dict[str, CollectionEnvironment]
     ) -> "MultiCollectionService":
         """
         Create a multi-collection service from an environment map.
 
         Args:
-            env_map: Map of display_name -> ResearchAgentEnvironment
+            env_map: Map of display_name -> CollectionEnvironment
 
         Returns:
             MultiCollectionService instance managing multiple collections
@@ -477,19 +477,19 @@ class MultiCollectionService:
     methods to work with all collections at once.
     """
 
-    def __init__(self, env_map: Dict[str, ResearchAgentEnvironment]):
+    def __init__(self, env_map: Dict[str, CollectionEnvironment]):
         """
         Initialize the multi-collection service.
 
         Args:
-            env_map: Map of collection_name -> ResearchAgentEnvironment
+            env_map: Map of display_name -> CollectionEnvironment
         """
         self.services: Dict[str, CollectionService] = {}
-        for collection_name, system_env in env_map.items():
-            # Use collection_name as both internal name and display name
-            self.services[collection_name] = CollectionService(
-                collection_name=collection_name,
-                display_name=collection_name,
+        for display_name, system_env in env_map.items():
+            # Key by sanitized collection_name for API lookups, but preserve display_name
+            self.services[system_env.collection_name] = CollectionService(
+                collection_name=system_env.collection_name,
+                display_name=display_name,
                 system_env=system_env,
             )
 

docs/reference/language-grammar.md

@@ -52,25 +52,34 @@ embedding_batch: "BATCH" "(" embedding_batch_item ("," embedding_batch_item)* ")
 
 ```
 where_clause: "WHERE" predicate
-where_document_clause: "WHERE_DOCUMENT" document_predicate
+where_document_clause: "WHERE_DOCUMENT" document_predicate_expr
 
 predicate:
     or_expr
 
-document_predicate:
+document_predicate_expr:
+    document_or_expr
+document_or_expr:
+    document_and_expr ("OR" document_and_expr)*
+document_and_expr:
+    document_atom ("AND" document_atom)*
+document_atom:
+    | "(" document_predicate_expr ")"
     | "CONTAINS" value
     | "LIKE" string_literal
+    | "document" "CONTAINS" value
+    | "document" "LIKE" string_literal
 ```
 
 Metadata predicates support:
 
 - Comparisons (`=`, `!=`, `<`, `<=`, `>`, `>=`)
 - `IN` / `NOT IN`
 - `BETWEEN`
-- `LIKE` (`%value%` form)
-- `CONTAINS`
 
-Boolean expressions use `AND` / `OR` with parentheses for grouping.
+**Note:** `LIKE` and `CONTAINS` are only supported for document predicates (via `WHERE_DOCUMENT`), not for metadata filters. This is a ChromaDB limitation.
+
+Both `WHERE` and `WHERE_DOCUMENT` support boolean expressions with `AND` / `OR` and parentheses for grouping.
 
 ## Similarity & TopK
 

docs/reference/multi-collection-architecture.md

@@ -146,9 +146,9 @@ query_config.json maps models → collections
 from pathlib import Path
 from chromasql.adapters import MetadataFieldRouter
 from chromasql.multi_collection import execute_multi_collection
-from indexer.query_lib.async_multi_collection_adapter import AsyncMultiCollectionAdapter
-from indexer.vectorize_lib.query_client import AsyncMultiCollectionQueryClient
-from indexer.vectorize_lib.query_config import load_query_config
+from idxr.query_lib.async_multi_collection_adapter import AsyncMultiCollectionAdapter
+from idxr.vectorize_lib.query_client import AsyncMultiCollectionQueryClient
+from idxr.vectorize_lib.query_config import load_query_config
 
 # Load config
 config = load_query_config(Path("output/query_config.json"))

docs/using-chromasql/quick-reference.md

@@ -33,23 +33,63 @@ omitting it produces a metadata-only `get`.
 
 ### Metadata (`WHERE`)
 
-Supports equality, inequality, numeric comparisons, `IN`/`NOT IN`, `BETWEEN`,
-`LIKE` (with `%value%` pattern), and `CONTAINS`.
+Supports equality, inequality, numeric comparisons, `IN`/`NOT IN`, and `BETWEEN`.
+
+**Note:** `LIKE` and `CONTAINS` are NOT supported on metadata (ChromaDB limitation).
+Use `WHERE_DOCUMENT` for text/pattern matching.
+
+**Note:** `BETWEEN` with mixed int/float types may behave unexpectedly due to ChromaDB type coercion. Use matching types (integer boundaries for integer metadata).
+
+**Important:** Different filter types (metadata vs. document) can **only** be combined with `AND`, not `OR` (ChromaDB limitation). Within each type, use `OR` freely.
 
 ```sql
+-- Valid: metadata AND document
 WHERE metadata.category = 'outerwear'
-  AND metadata.tags CONTAINS 'waterproof'
+  AND metadata.price BETWEEN 50 AND 150
+  AND document CONTAINS 'waterproof'
+
+-- Invalid: metadata OR document ❌
+-- WHERE metadata.category = 'outerwear' OR document CONTAINS 'sale'
 ```
 
 ### Document (`WHERE_DOCUMENT`)
 
-Applied after metadata filters. Supports `CONTAINS` and `%value%` `LIKE`
-patterns.
+Applied after metadata filters. Supports text search operators: `CONTAINS`, `NOT CONTAINS`, `LIKE`, `NOT LIKE`, `REGEX`, `NOT REGEX`.
+
+**These operators ONLY work with WHERE_DOCUMENT**, not with WHERE (metadata).
+
+**Boolean expressions supported**: Use `AND`, `OR`, and parentheses for complex filters.
+
+**Important:** Use `WHERE_DOCUMENT` **once** at the beginning, then combine predicates with boolean operators. Don't repeat `WHERE_DOCUMENT` for each condition.
 
 ```sql
+-- Simple filter
 WHERE_DOCUMENT LIKE '%gore-tex%'
+
+-- OR: Match multiple terms (don't repeat WHERE_DOCUMENT!)
+WHERE_DOCUMENT CONTAINS 'waterproof' OR CONTAINS 'breathable'
+
+-- AND: Match all terms
+WHERE_DOCUMENT CONTAINS 'outdoor' AND LIKE '%jacket%'
+
+-- Complex: Nested with parentheses
+WHERE_DOCUMENT (CONTAINS 'outdoor' AND LIKE '%jacket%') OR CONTAINS 'windproof'
+
+-- Real-world: Multiple organization names
+WHERE_DOCUMENT CONTAINS 'BofA' OR CONTAINS 'Bank of America' OR LIKE '%Wells Fargo%'
+
+-- Exclude patterns
+WHERE_DOCUMENT NOT LIKE '%test%'
+WHERE_DOCUMENT NOT CONTAINS 'deprecated'
+
+-- Regex patterns
+WHERE_DOCUMENT REGEX '[a-z]+@[a-z]+\.com'  -- Email pattern
+WHERE_DOCUMENT REGEX '(?i)python'  -- Case-insensitive matching
+WHERE_DOCUMENT NOT REGEX '\d{3}-\d{2}-\d{4}'  -- Exclude SSN patterns
 ```
 
+**Note:** Text operators are **case-sensitive**. `WHERE_DOCUMENT CONTAINS 'urgent'` will NOT match "Urgent" or "URGENT". To handle multiple cases, use OR: `CONTAINS 'urgent' OR CONTAINS 'Urgent'`, or use REGEX with `(?i)` flag: `REGEX '(?i)urgent'`.
+
 ## Embedding Clauses
 
 ```sql