diff --git a/notebooks/advanced_techniques/instruction-following-reranking.ipynb b/notebooks/advanced_techniques/instruction-following-reranking.ipynb new file mode 100644 index 0000000..18a4c77 --- /dev/null +++ b/notebooks/advanced_techniques/instruction-following-reranking.ipynb @@ -0,0 +1,530 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "24e92416", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mongodb-developer/GenAI-Showcase/blob/main/notebooks/advanced_techniques/instruction-following-reranking.ipynb)\n", + "\n", + "[![View Article](https://img.shields.io/badge/View%20Article-blue)](https://www.mongodb.com/developer/products/atlas/parent-doc-retrieval/?utm_campaign=devrel&utm_source=cross-post&utm_medium=organic_social&utm_content=https%3A%2F%2Fgithub.com%2Fmongodb-developer%2FGenAI-Showcase&utm_term=apoorva.joshi)" + ] + }, + { + "cell_type": "markdown", + "id": "c4940a27", + "metadata": {}, + "source": [ + "# Using Instruction-following Reranking in your AI Applications\n", + "\n", + "This notebook shows you how to implement instruction-following reranking in your AI applications using Voyage AI's `reranker-2.5` model." + ] + }, + { + "cell_type": "markdown", + "id": "2881869b", + "metadata": {}, + "source": [ + "## Step 1: Install required libraries\n", + "\n", + "- **voyageai**: Python library to interact with Voyage AI's APIs" + ] + }, + { + "cell_type": "markdown", + "id": "65b9b945", + "metadata": {}, + "source": [ + "## Step 2: Setup prerequisites\n", + "\n", + "Set the Voyage API key as an environment variable, and initialize the Voyage AI client.\n", + "\n", + "Steps to obtain a Voyage AI API Key can be found [here](https://docs.voyageai.com/docs/api-key-and-installation#authentication-with-api-keys)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8732ca31", + "metadata": {}, + "outputs": [], + "source": [ + "import getpass\n", + "import os\n", + "\n", + "import voyageai" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "374bc80f", + "metadata": {}, + "outputs": [], + "source": [ + "os.environ[\"VOYAGE_API_KEY\"] = getpass.getpass(\"Voyage API Key:\")\n", + "vo = voyageai.Client()" + ] + }, + { + "cell_type": "markdown", + "id": "2bc85a64", + "metadata": {}, + "source": [ + "## Scenario 1: Incorporating implicit business logic" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "4ced4770", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"How to treat migraines?\"" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "8867eecf", + "metadata": {}, + "outputs": [], + "source": [ + "# Documents retrieved from search\n", + "documents = [\n", + " \"Source: peer_reviewed_journal, A randomized controlled trial of 600 patients found topiramate reduced migraine frequency by 50% compared to placebo, with common side effects including cognitive slowing.\",\n", + " \"Source: general_website, Migraine prevention includes lifestyle changes and medications. Common preventive drugs include beta-blockers and antidepressants. Talk to your doctor about which option is right for you.\",\n", + " \"Source: forum, I started taking magnesium and B2 supplements for my migraines and they've totally disappeared! Haven't needed my prescription in months.\",\n", + " \"Source: peer_reviewed_journal, Systematic review of 42 studies shows CGRP monoclonal antibodies reduce monthly migraine days by 4-6 days on average, with better tolerability profiles than traditional preventives.\",\n", + " \"Source: general_website, 10 natural remedies for migraines that actually work! From essential oils to ice packs, these home treatments can help you avoid medication side effects.\",\n", + " \"Source: healthcare_provider, Preventive migraine medications work by reducing the frequency and severity of attacks. Options include daily pills, monthly injections, or quarterly infusions depending on your needs.\",\n", + " \"Source: peer_reviewed_journal, Meta-analysis of acupuncture trials demonstrates modest benefit for migraine prevention, with effect size comparable to some pharmacological interventions and minimal adverse events.\",\n", + "]" + ] + }, + { + "cell_type": "markdown", + "id": "0cd89418", + "metadata": {}, + "source": [ + "### With instruction-following" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e751a46b", + "metadata": {}, + "outputs": [], + "source": [ + "instructions = \"Prioritize peer-reviewed journals, followed by advice from healthcare providers, then general websites and finally forums.\"" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "71b95eac", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Document: Source: peer_reviewed_journal, A randomized controlled trial of 600 patients found topiramate reduced migraine frequency by 50% compared to placebo, with common side effects including cognitive slowing.\n", + "Score: 0.8203125\n", + "\n", + "Document: Source: peer_reviewed_journal, Systematic review of 42 studies shows CGRP monoclonal antibodies reduce monthly migraine days by 4-6 days on average, with better tolerability profiles than traditional preventives.\n", + "Score: 0.8203125\n", + "\n", + "Document: Source: peer_reviewed_journal, Meta-analysis of acupuncture trials demonstrates modest benefit for migraine prevention, with effect size comparable to some pharmacological interventions and minimal adverse events.\n", + "Score: 0.77734375\n", + "\n" + ] + } + ], + "source": [ + "# Use the rerank method to rerank retrieved results\n", + "reranking = vo.rerank(\n", + " query=f\"{instructions}\\nQuery: {query}\",\n", + " documents=documents,\n", + " model=\"rerank-2.5\",\n", + " top_k=3,\n", + ")\n", + "\n", + "# Print the reranking results\n", + "for r in reranking.results:\n", + " print(f\"Document: {r.document}\")\n", + " print(f\"Score: {r.relevance_score}\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "f32c66e4", + "metadata": {}, + "source": [ + "### Without instruction-following" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "60905c73", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Document: Source: forum, I started taking magnesium and B2 supplements for my migraines and they've totally disappeared! Haven't needed my prescription in months.\n", + "Score: 0.75390625\n", + "\n", + "Document: Source: general_website, 10 natural remedies for migraines that actually work! From essential oils to ice packs, these home treatments can help you avoid medication side effects.\n", + "Score: 0.75\n", + "\n", + "Document: Source: healthcare_provider, Preventive migraine medications work by reducing the frequency and severity of attacks. Options include daily pills, monthly injections, or quarterly infusions depending on your needs.\n", + "Score: 0.75\n", + "\n" + ] + } + ], + "source": [ + "# Use the rerank method to rerank retrieved results\n", + "reranking = vo.rerank(query=query, documents=documents, model=\"rerank-2.5\", top_k=3)\n", + "\n", + "# Print the reranking results\n", + "for r in reranking.results:\n", + " print(f\"Document: {r.document}\")\n", + " print(f\"Score: {r.relevance_score}\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "84a2d8a8", + "metadata": {}, + "source": [ + "## Scenario 2: Handling different types of queries" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "f7044032", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"My query is not using the index I created\"" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "e09751a7", + "metadata": {}, + "outputs": [], + "source": [ + "# Documents retrieved from search\n", + "documents = [\n", + " # Beginner/Conceptual content about indexes\n", + " \"What are MongoDB indexes? Indexes are special data structures that store a small portion of your data in an easy-to-traverse form. Like a book's index helps you find topics without reading every page, MongoDB indexes help find documents without scanning the entire collection.\",\n", + " \"How indexes improve query performance: Without an index, MongoDB must scan every document in a collection (a collection scan) to find matches. With an index, MongoDB can limit the number of documents it must inspect. For a collection with millions of documents, this means the difference between a query taking seconds versus milliseconds.\",\n", + " \"Understanding index types: MongoDB supports several index types. Single field indexes are the simplest, indexing one field. Compound indexes index multiple fields together and are useful when queries filter on multiple fields. Text indexes enable text search capabilities, while geospatial indexes support location-based queries.\",\n", + " \"The tradeoffs of indexing: While indexes speed up read operations, they slow down writes because MongoDB must update indexes whenever documents change. Indexes also consume disk space and memory. It's important to create indexes strategically for your most common queries rather than indexing every field.\",\n", + " # Implementation content about indexes\n", + " \"Creating a single field index: ```javascript\\ndb.users.createIndex({ email: 1 })\\n// 1 for ascending, -1 for descending\\ndb.products.createIndex({ price: -1 })``` The sort order matters for sorting queries but not for equality matches.\",\n", + " \"Building compound indexes: ```javascript\\ndb.orders.createIndex({ user_id: 1, created_at: -1 })\\n// This supports queries on user_id alone, or user_id + created_at\\n// But NOT queries on created_at alone``` Field order matters - most selective field should come first.\",\n", + " \"Creating unique indexes: ```javascript\\ndb.users.createIndex({ email: 1 }, { unique: true })\\n// Prevents duplicate email addresses\\ndb.sessions.createIndex({ token: 1 }, { unique: true, sparse: true })``` Sparse indexes only include documents with the indexed field.\",\n", + " \"Adding indexes with options: ```javascript\\ndb.logs.createIndex(\\n { created_at: 1 },\\n { expireAfterSeconds: 86400 } // TTL index, auto-deletes after 24 hours\\n)\\ndb.large_collection.createIndex({ status: 1 }, { background: true })``` Background builds don't block database operations.\",\n", + " \"Creating text indexes for search: ```javascript\\ndb.articles.createIndex({ title: 'text', content: 'text' })\\n// Query with:\\ndb.articles.find({ $text: { $search: 'mongodb performance' } }).sort({ score: { $meta: 'textScore' } })``` Only one text index per collection allowed.\",\n", + " # Troubleshooting content about indexes\n", + " \"Query not using an index? Use explain() to diagnose: ```javascript\\ndb.users.find({ email: 'user@example.com' }).explain('executionStats')``` Look for 'IXSCAN' (good) vs 'COLLSCAN' (bad). If you see COLLSCAN, create an index on the queried field.\",\n", + " \"Index build failing or taking too long? Quick fixes: Use {background: true} option to avoid blocking writes. For large collections, build indexes during low-traffic periods. On replica sets, use rolling index builds: build on secondaries first, then step down primary and build there.\",\n", + " \"Too many indexes slowing down writes? Solution: Audit your indexes with db.collection.getIndexes() and remove unused ones. Use MongoDB's index usage stats: ```javascript\\ndb.collection.aggregate([{ $indexStats: {} }])``` Drop indexes with zero or minimal 'ops' count.\",\n", + " \"Index taking too much memory? Check index size with db.collection.stats() and look at 'indexSizes'. Solutions: 1) Drop unused indexes 2) Use partial indexes to index only relevant documents: ```javascript\\ndb.orders.createIndex({ status: 1 }, { partialFilterExpression: { status: { $ne: 'archived' } } })```\",\n", + " \"Compound index not being used? Verify your query matches the index prefix. An index on {a: 1, b: 1, c: 1} supports queries on 'a', 'a+b', and 'a+b+c', but NOT 'b' or 'c' alone. Reorder index fields or create additional indexes for different query patterns.\",\n", + "]" + ] + }, + { + "cell_type": "markdown", + "id": "efa75bd6", + "metadata": {}, + "source": [ + "### With instruction-following" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2247ed0f", + "metadata": {}, + "outputs": [], + "source": [ + "# Say the query has been classified as \"troubleshooting\"\n", + "query_type = \"troubleshooting\"\n", + "# Generate instructions based on query type\n", + "if query_type == \"concepts\":\n", + " instructions = \"Prioritize explanatory content.\"\n", + "elif query_type == \"implementation\":\n", + " instructions = \"Prioritize code examples.\"\n", + "elif query_type == \"troubleshooting\":\n", + " instructions = \"Prioritize quickfixes and step-by-step debugging instructions.\"" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "053ffb9f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Document: Query not using an index? Use explain() to diagnose: ```javascript\n", + "db.users.find({ email: 'user@example.com' }).explain('executionStats')``` Look for 'IXSCAN' (good) vs 'COLLSCAN' (bad). If you see COLLSCAN, create an index on the queried field.\n", + "Score: 0.75\n", + "\n", + "Document: Compound index not being used? Verify your query matches the index prefix. An index on {a: 1, b: 1, c: 1} supports queries on 'a', 'a+b', and 'a+b+c', but NOT 'b' or 'c' alone. Reorder index fields or create additional indexes for different query patterns.\n", + "Score: 0.67578125\n", + "\n", + "Document: Index build failing or taking too long? Quick fixes: Use {background: true} option to avoid blocking writes. For large collections, build indexes during low-traffic periods. On replica sets, use rolling index builds: build on secondaries first, then step down primary and build there.\n", + "Score: 0.546875\n", + "\n" + ] + } + ], + "source": [ + "# Use the rerank method to rerank retrieved results\n", + "reranking = vo.rerank(\n", + " query=f\"{instructions}\\nQuery: {query}\",\n", + " documents=documents,\n", + " model=\"rerank-2.5\",\n", + " top_k=3,\n", + ")\n", + "\n", + "# Print the reranking results\n", + "for r in reranking.results:\n", + " print(f\"Document: {r.document}\")\n", + " print(f\"Score: {r.relevance_score}\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "a4f334fe", + "metadata": {}, + "source": [ + "### Without instruction-following" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "8522d06c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Document: Query not using an index? Use explain() to diagnose: ```javascript\n", + "db.users.find({ email: 'user@example.com' }).explain('executionStats')``` Look for 'IXSCAN' (good) vs 'COLLSCAN' (bad). If you see COLLSCAN, create an index on the queried field.\n", + "Score: 0.80078125\n", + "\n", + "Document: Compound index not being used? Verify your query matches the index prefix. An index on {a: 1, b: 1, c: 1} supports queries on 'a', 'a+b', and 'a+b+c', but NOT 'b' or 'c' alone. Reorder index fields or create additional indexes for different query patterns.\n", + "Score: 0.76953125\n", + "\n", + "Document: Too many indexes slowing down writes? Solution: Audit your indexes with db.collection.getIndexes() and remove unused ones. Use MongoDB's index usage stats: ```javascript\n", + "db.collection.aggregate([{ $indexStats: {} }])``` Drop indexes with zero or minimal 'ops' count.\n", + "Score: 0.6171875\n", + "\n" + ] + } + ], + "source": [ + "# Use the rerank method to rerank retrieved results\n", + "reranking = vo.rerank(query=query, documents=documents, model=\"rerank-2.5\", top_k=3)\n", + "\n", + "# Print the reranking results\n", + "for r in reranking.results:\n", + " print(f\"Document: {r.document}\")\n", + " print(f\"Score: {r.relevance_score}\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "00d8dd5f", + "metadata": {}, + "source": [ + "## Scenario 3: Managing long-term memories and state" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "7403ec36", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"Vacation ideas in March for me and my husband\"" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "938d2448", + "metadata": {}, + "outputs": [], + "source": [ + "# Documents retrieved from semantic search-- formatted\n", + "documents = [\n", + " # Recent luxury bookings (2024)\n", + " \"Date: 2025-09-15, Booked Ritz-Carlton Maui, oceanfront suite, $850/night. Anniversary trip with husband David.\",\n", + " \"Date: 2025-09-20, Complained about noise from pool area at Ritz-Carlton Maui. Requested quiet rooms for future stays.\",\n", + " \"Date: 2025-07-10, Booked Four Seasons Bora Bora, overwater bungalow, $1200/night. Solo work retreat.\",\n", + " # Critical constraint\n", + " \"Date: 2025-06-01, Customer has severe shellfish allergy. Avoid destinations where seafood is primary cuisine.\",\n", + " # Recent preferences\n", + " \"Date: 2025-09-16, Prefers hotels with spa facilities. Mentioned spa treatments are essential for relaxation.\",\n", + " \"Date: 2025-08-30, Achieved Marriott Bonvoy Gold status. Prefers Marriott properties for loyalty benefits.\",\n", + " \"Date: 2025-08-05, Inquired about beach destinations in Mexico and Caribbean for March 2025. Budget: up to $1000/night.\",\n", + " # General preference\n", + " \"Date: 2024-01-15, Prefers warm beach destinations year-round. Enjoys water activities and relaxation.\",\n", + " # Older budget bookings (2023)\n", + " \"Date: 2023-11-05, Booked Holiday Inn Orlando, $120/night. Family trip with two children.\",\n", + " \"Date: 2023-03-10, Booked Hampton Inn Chicago, $95/night. Budget business trip.\",\n", + " \"Date: 2022-12-01, Booked Motel 6 Las Vegas, $65/night. Road trip with friends.\",\n", + " \"Date: 2022-06-15, Mentioned preferring budget accommodations to save money for activities.\",\n", + "]" + ] + }, + { + "cell_type": "markdown", + "id": "2daa2b10", + "metadata": {}, + "source": [ + "### With instruction-following" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "12953970", + "metadata": {}, + "outputs": [], + "source": [ + "instructions = \"Prioritize recent booking patterns (last 12 months), safety concerns, and dietary restrictions.\"" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "id": "72a171bf", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Document: Date: 2025-08-05, Inquired about beach destinations in Mexico and Caribbean for March 2025. Budget: up to $1000/night.\n", + "Relevance Score: 0.5078125\n", + "\n", + "Document: Date: 2025-06-01, Customer has severe shellfish allergy. Avoid destinations where seafood is primary cuisine.\n", + "Relevance Score: 0.404296875\n", + "\n", + "Document: Date: 2025-09-15, Booked Ritz-Carlton Maui, oceanfront suite, $850/night. Anniversary trip with husband David.\n", + "Relevance Score: 0.404296875\n", + "\n" + ] + } + ], + "source": [ + "reranking = vo.rerank(\n", + " query=f\"{instructions}\\nQuery: {query}\",\n", + " documents=documents,\n", + " model=\"rerank-2.5\",\n", + " top_k=3,\n", + ")\n", + "\n", + "for r in reranking.results:\n", + " print(f\"Document: {r.document}\")\n", + " print(f\"Relevance Score: {r.relevance_score}\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "e9a380ea", + "metadata": {}, + "source": [ + "### Without instruction-following" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "id": "5233c6a2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Document: Date: 2025-08-05, Inquired about beach destinations in Mexico and Caribbean for March 2025. Budget: up to $1000/night.\n", + "Relevance Score: 0.6796875\n", + "\n", + "Document: Date: 2024-01-15, Prefers warm beach destinations year-round. Enjoys water activities and relaxation.\n", + "Relevance Score: 0.4765625\n", + "\n", + "Document: Date: 2025-09-15, Booked Ritz-Carlton Maui, oceanfront suite, $850/night. Anniversary trip with husband David.\n", + "Relevance Score: 0.47265625\n", + "\n" + ] + } + ], + "source": [ + "reranking = vo.rerank(\n", + " query=query,\n", + " documents=documents,\n", + " model=\"rerank-2.5\",\n", + " top_k=3,\n", + ")\n", + "\n", + "for r in reranking.results:\n", + " print(f\"Document: {r.document}\")\n", + " print(f\"Relevance Score: {r.relevance_score}\")\n", + " print()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.1" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "state": {} + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}