Add files via upload

Author: BBC-Esq

src/chat_kobold.py

@@ -47,7 +47,7 @@ def connect_to_kobold(self, augmented_query):
 
         response = None
         try:
-            response = requests.post(self.api_url, json=payload, stream=True)
+            response = requests.post(self.api_url, json=payload, stream=True, timeout=20)
             response.raise_for_status()
             client = sseclient.SSEClient(response)
 

src/database_interactions.py

@@ -1,6 +1,10 @@
 import gc
 import os
+
+os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
+
 import time
+import json
 from copy import deepcopy
 from pathlib import Path
 from typing import Optional
@@ -26,33 +30,6 @@
 from constants import VECTOR_MODELS
 
 
-def _flatten_to_text(x):
-    if x is None:
-        return ""
-    if isinstance(x, str):
-        return x
-    if isinstance(x, bytes):
-        try:
-            return x.decode("utf-8", "ignore")
-        except Exception:
-            return ""
-    if isinstance(x, (bool, int, float)):
-        return str(x)
-    if hasattr(x, '__iter__') and not isinstance(x, (str, bytes)):
-        parts = []
-        try:
-            for item in x:
-                s = _flatten_to_text(item)
-                if s:
-                    parts.append(s)
-            return " ".join(parts)
-        except Exception:
-            return str(x)
-    try:
-        return str(x)
-    except Exception:
-        return ""
-
 class BaseEmbeddingModel:
 
     def __init__(self, model_name, model_kwargs, encode_kwargs, is_query: bool = False):
@@ -70,7 +47,7 @@ def prepare_kwargs(self):
         tok_kw = hf_embed_kw.setdefault("tokenizer_kwargs", {})
         tok_kw.update({
             "trust_remote_code": True,
-            "use_fast": True,
+            "use_fast": False,
             "model_max_length": 512,
         })
 
@@ -95,10 +72,10 @@ def prepare_encode_kwargs(self):
         encode_kwargs.setdefault("convert_to_tensor", False)
         encode_kwargs.setdefault("show_progress_bar", not self.is_query)
 
-        params_to_remove = ['model_max_length', 'return_token_type_ids', 'show_progress_bar', 
-                            'padding', 'truncation', 'max_length']
+        encode_kwargs.setdefault("padding", True)
+        encode_kwargs.setdefault("truncation", True)
 
-        for param in params_to_remove:
+        for param in ['model_max_length', 'return_token_type_ids', 'show_progress_bar']:
             encode_kwargs.pop(param, None)
 
         return encode_kwargs
@@ -360,29 +337,48 @@ def create_database(self, texts, embeddings):
             chunk_counters = defaultdict(int)
             skipped_chunks = 0
 
-            for idx, doc in enumerate(texts):
+            for original_idx, doc in enumerate(texts):
+                raw_content = None
+                clean_content = None
+
                 try:
                     if hasattr(doc, 'page_content'):
                         raw_content = doc.page_content
                     else:
                         raw_content = doc
-                    
+
                     if raw_content is None:
+                        print(f"Skipping chunk {original_idx}: no content")
+                        skipped_chunks += 1
+                        continue
+
+                    if isinstance(raw_content, (list, tuple, dict)):
+                        print(f"Skipping chunk {original_idx}: invalid type {type(raw_content)}")
                         skipped_chunks += 1
                         continue
 
                     if not isinstance(raw_content, str):
-                        raw_content = str(raw_content)
+                        try:
+                            raw_content = str(raw_content)
+                        except Exception as e:
+                            print(f"Skipping chunk {original_idx}: cannot coerce {type(raw_content)} to str ({e})")
+                            skipped_chunks += 1
+                            continue
 
                     clean_content = raw_content.replace('\x00', ' ')
+                    clean_content = ' '.join(clean_content.split())
 
-                    words = clean_content.split()
-                    clean_content = ' '.join(words)
+                    if not clean_content or not clean_content.strip():
+                        skipped_chunks += 1
+                        continue
 
-                    if not clean_content:
+                    if isinstance(clean_content, (list, tuple, dict)):
+                        print(f"Skipping chunk {original_idx}: invalid type after clean {type(clean_content)}")
                         skipped_chunks += 1
                         continue
 
+                    clean_content.encode('utf-8')
+
                     file_hash = doc.metadata.get('hash') if hasattr(doc, 'metadata') else None
                     chunk_counters[file_hash] += 1
                     tiledb_id = str(random.randint(0, MAX_UINT64 - 1))
@@ -391,9 +387,19 @@ def create_database(self, texts, embeddings):
                     all_metadatas.append(doc.metadata if hasattr(doc, 'metadata') else {})
                     all_ids.append(tiledb_id)
                     hash_id_mappings.append((tiledb_id, file_hash))
-                    
+
                 except Exception as e:
-                    print(f"Error processing chunk {idx}: {e}")
+                    preview = None
+                    try:
+                        preview = (clean_content if isinstance(clean_content, str) else raw_content)
+                        if isinstance(preview, str):
+                            preview = preview[:120].replace('\n', ' ')
+                        else:
+                            preview = repr(preview)[:120]
+                    except Exception:
+                        preview = "<unavailable>"
+
+                    print(f"Error processing chunk {original_idx}: {e} | preview: {preview}")
                     skipped_chunks += 1
                     continue
 
@@ -410,6 +416,10 @@ def create_database(self, texts, embeddings):
 
             embedding_start_time = time.time()
 
+            for i, t in enumerate(all_texts):
+                if not isinstance(t, str):
+                    raise TypeError(f"Non-string at index {i}: {type(t)}")
+
             vectors = embeddings.embed_documents(all_texts)
 
             embedding_end_time = time.time()
@@ -486,12 +496,12 @@ def create_metadata_db(self, documents, hash_id_mappings):
                 for doc in documents
             ]
             cursor.executemany('''
-                INSERT INTO document_metadata (file_name, hash, file_path, page_content)
+                INSERT OR REPLACE INTO document_metadata (file_name, hash, file_path, page_content)
                 VALUES (?, ?, ?, ?)
             ''', doc_rows)
 
             cursor.executemany('''
-                INSERT INTO hash_chunk_ids (tiledb_id, hash)
+                INSERT OR REPLACE INTO hash_chunk_ids (tiledb_id, hash)
                 VALUES (?, ?)
             ''', hash_id_mappings)
 
@@ -727,7 +737,7 @@ def search(self, query, k: Optional[int] = None, score_threshold: Optional[float
         if search_term:
             filtered_contexts = [
                 (doc, score) for doc, score in relevant_contexts
-                if search_term in str(doc.page_content).lower()
+                if search_term not in str(doc.page_content).lower()
             ]
         else:
             filtered_contexts = relevant_contexts

src/document_processor.py

@@ -27,22 +27,14 @@
     BSHTMLLoader
 )
 
-from typing import Optional, Any, Iterator, Union
+from typing import Optional, Any, Iterator, Union, List
 from langchain_community.document_loaders.blob_loaders import Blob
 from langchain_community.document_loaders.parsers import PyMuPDFParser
 import pymupdf
 
 from constants import DOCUMENT_LOADERS
 from extract_metadata import extract_document_metadata, add_pymupdf_page_metadata, compute_content_hash
 
-# logging.basicConfig(
-    # level=logging.ERROR,
-    # format='%(asctime)s - %(levelname)s - %(message)s',
-    # handlers=[
-        # logging.FileHandler('document_processor.log', mode='w')
-    # ]
-# )
-
 warnings.filterwarnings("ignore", category=FutureWarning)
 warnings.filterwarnings("ignore", category=UserWarning)
 
@@ -51,33 +43,25 @@
 INGEST_THREADS = max(2, os.cpu_count() - 12)
 
 
-from typing import List
-
 class FixedSizeTextSplitter:
-    """Splits text into equally-sized character chunks.
-
-    Parameters
-    ----------
-    chunk_size : int
-        Maximum characters per chunk.  Taken from config.yaml.
-    """
 
-    def __init__(self, chunk_size: int):
+    def __init__(self, chunk_size: int, chunk_overlap: int = 0):
         self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
 
     def split_documents(self, docs: List[Document]) -> List[Document]:
         chunks: List[Document] = []
         for doc in docs:
             text = doc.page_content or ""
-            for start in range(0, len(text), self.chunk_size):
-                piece = text[start : start + self.chunk_size].strip()
-                if not piece:
-                    continue
-                # shallow-copy metadata so each chunk carries origin info
-                chunks.append(Document(page_content=piece, metadata=dict(doc.metadata)))
+            start = 0
+            while start < len(text):
+                end = start + self.chunk_size
+                piece = text[start:end].strip()
+                if piece:
+                    chunks.append(Document(page_content=piece, metadata=dict(doc.metadata)))
+                start += self.chunk_size - self.chunk_overlap
         return chunks
 
-
 class CustomPyMuPDFParser(PyMuPDFParser):
     def _lazy_parse(self, blob: Blob, text_kwargs: Optional[dict[str, Any]] = None) -> Iterator[Document]:
         with PyMuPDFParser._lock:
@@ -103,7 +87,6 @@ def __init__(self, file_path: Union[str, PurePath], **kwargs: Any) -> None:
             extract_images=kwargs.get('extract_images', False)
         )
 
-# map loaders
 for ext, loader_name in DOCUMENT_LOADERS.items():
     DOCUMENT_LOADERS[ext] = globals()[loader_name]
 
@@ -208,7 +191,6 @@ def load_documents(source_dir: Path) -> list:
     if doc_paths:
         n_workers = min(INGEST_THREADS, max(len(doc_paths), 1))
 
-        total_cores = os.cpu_count()
         threads_per_process = 2
 
         with ProcessPoolExecutor(n_workers) as executor:
@@ -228,24 +210,17 @@ def split_documents(documents=None, text_documents_pdf=None):
    try:
        print("\nSplitting documents into chunks.")
 
-       with open("config.yaml", "r", encoding='utf-8') as config_file:
+       config_path = Path(__file__).resolve().parent / "config.yaml"
+       with open(config_path, "r", encoding='utf-8') as config_file:
            config = yaml.safe_load(config_file)
            chunk_size = config["database"]["chunk_size"]
            chunk_overlap = config["database"]["chunk_overlap"]
 
-       # instantiate text splitter
-       text_splitter = FixedSizeTextSplitter(chunk_size=chunk_size)
-
-       # text_splitter = RecursiveCharacterTextSplitter(
-           # chunk_size=chunk_size,
-           # chunk_overlap=chunk_overlap,
-           # keep_separator=False,
-       # )
+       text_splitter = FixedSizeTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
 
        texts = []
 
        if documents:
-           # use text splitter directly
            texts = text_splitter.split_documents(documents)
 
        if text_documents_pdf:
@@ -265,4 +240,4 @@ def split_documents(documents=None, text_documents_pdf=None):
    except Exception as e:
        logging.exception("Error during document splitting")
        logging.error(f"Error type: {type(e)}")
-       raise
+       raise

src/extract_metadata.py

@@ -32,20 +32,15 @@ def extract_common_metadata(file_path, content_hash=None):
         "modification_date": modification_date,
         "hash": file_hash
     }
-    
-    #=========================================================================
+
     clean_metadata = {}
     for k, v in metadata.items():
         if isinstance(v, (str, int, float, bool, type(None))):
             clean_metadata[k] = v
-        elif isinstance(v, enumerate):
-            print(f"❌ ENUMERATE in metadata key '{k}' - converting to string")
-            clean_metadata[k] = str(list(v))
         else:
             clean_metadata[k] = str(v)
-    
+
     return clean_metadata
-    #=========================================================================
 
 def extract_image_metadata(file_path):
     metadata = extract_common_metadata(file_path)
@@ -81,21 +76,21 @@ def split_text(text: str, chunk_size: int, chunk_overlap: int) -> List[Tuple[str
                 end = len(clean_text)
             chunk = clean_text[start:end].strip()
 
-            page_num = None
+            page_num = 1
             for marker_pos, page in reversed(page_markers):
                 if marker_pos <= start:
                     page_num = page
                     break
 
-            if chunk and page_num is not None:
+            if chunk:
                 chunks.append((chunk, page_num))
+            
             start += chunk_size - chunk_overlap
 
         return chunks
 
     chunks = split_text(doc.page_content, chunk_size, chunk_overlap)
 
-    #================================================================================================
     new_docs = []
     for chunk, page_num in chunks:
         new_metadata = {}
@@ -105,9 +100,6 @@ def split_text(text: str, chunk_size: int, chunk_overlap: int) -> List[Tuple[str
                     key = str(k)
                     if isinstance(v, (str, int, float, bool)):
                         new_metadata[key] = v
-                    elif isinstance(v, enumerate):
-                        print(f"❌ ENUMERATE in chunk metadata key '{key}' - converting to string")
-                        new_metadata[key] = str(list(v))
                     else:
                         new_metadata[key] = str(v)
 
@@ -118,6 +110,5 @@ def split_text(text: str, chunk_size: int, chunk_overlap: int) -> List[Tuple[str
             metadata=new_metadata
         )
         new_docs.append(new_doc)
-        #================================================================================================
 
     return new_docs