fix: Add URL filtering logic to ensure links start with the specified base URL.

Author: anand-presidio

src/core/experts/DocumentProcessor.ts

@@ -265,6 +265,7 @@ export class DocumentProcessor {
 				maxDepth,
 				maxPages,
 				timeout: crawlTimeout,
+				urlFilter: (link) => link.startsWith(url),
 				onPageCrawlComplete: async (data: CrawlResult) => {
 					// Forward to vector store manager using the instance reference
 					if (self.onCrawlComplete) {