Merge pull request #4 from pamelafox/evalsci

Evalsci

Author: pamelafox

.github/workflows/azure-dev.yml

@@ -69,6 +69,12 @@ jobs:
       AZURE_OPENAI_GPT4V_DEPLOYMENT_CAPACITY: ${{ vars.AZURE_OPENAI_GPT4V_DEPLOYMENT_CAPACITY }}
       AZURE_OPENAI_GPT4V_DEPLOYMENT_VERSION: ${{ vars.AZURE_OPENAI_GPT4V_DEPLOYMENT_VERSION }}
       AZURE_OPENAI_GPT4V_DEPLOYMENT_SKU: ${{ vars.AZURE_OPENAI_GPT4V_DEPLOYMENT_SKU }}
+      USE_EVAL: ${{ vars.USE_EVAL }}
+      AZURE_OPENAI_EVAL_MODEL: ${{ vars.AZURE_OPENAI_EVAL_MODEL }}
+      AZURE_OPENAI_EVAL_MODEL_VERSION: ${{ vars.AZURE_OPENAI_EVAL_MODEL_VERSION }}
+      AZURE_OPENAI_EVAL_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EVAL_DEPLOYMENT }}
+      AZURE_OPENAI_EVAL_DEPLOYMENT_SKU: ${{ vars.AZURE_OPENAI_EVAL_DEPLOYMENT_SKU }}
+      AZURE_OPENAI_EVAL_DEPLOYMENT_CAPACITY: ${{ vars.AZURE_OPENAI_EVAL_DEPLOYMENT_CAPACITY }}
       AZURE_OPENAI_DISABLE_KEYS: ${{ vars.AZURE_OPENAI_DISABLE_KEYS }}
       OPENAI_HOST: ${{ vars.OPENAI_HOST }}
       OPENAI_API_KEY: ${{ vars.OPENAI_API_KEY }}

.github/workflows/evaluate.yaml

@@ -3,7 +3,6 @@ name: Evaluate RAG answer flow
 on:
   issue_comment:
     types: [created]
-  workflow_dispatch:
 
 # Set up permissions for deploying with secretless Azure federated credentials
 # https://learn.microsoft.com/azure/developer/github/connect-from-azure?tabs=azure-portal%2Clinux#set-up-azure-login-with-openid-connect-authentication
@@ -68,6 +67,12 @@ jobs:
       AZURE_OPENAI_GPT4V_DEPLOYMENT_CAPACITY: ${{ vars.AZURE_OPENAI_GPT4V_DEPLOYMENT_CAPACITY }}
       AZURE_OPENAI_GPT4V_DEPLOYMENT_VERSION: ${{ vars.AZURE_OPENAI_GPT4V_DEPLOYMENT_VERSION }}
       AZURE_OPENAI_GPT4V_DEPLOYMENT_SKU: ${{ vars.AZURE_OPENAI_GPT4V_DEPLOYMENT_SKU }}
+      USE_EVAL: ${{ vars.USE_EVAL }}
+      AZURE_OPENAI_EVAL_MODEL: ${{ vars.AZURE_OPENAI_EVAL_MODEL }}
+      AZURE_OPENAI_EVAL_MODEL_VERSION: ${{ vars.AZURE_OPENAI_EVAL_MODEL_VERSION }}
+      AZURE_OPENAI_EVAL_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EVAL_DEPLOYMENT }}
+      AZURE_OPENAI_EVAL_DEPLOYMENT_SKU: ${{ vars.AZURE_OPENAI_EVAL_DEPLOYMENT_SKU }}
+      AZURE_OPENAI_EVAL_DEPLOYMENT_CAPACITY: ${{ vars.AZURE_OPENAI_EVAL_DEPLOYMENT_CAPACITY }}
       AZURE_OPENAI_DISABLE_KEYS: ${{ vars.AZURE_OPENAI_DISABLE_KEYS }}
       OPENAI_HOST: ${{ vars.OPENAI_HOST }}
       OPENAI_API_KEY: ${{ vars.OPENAI_API_KEY }}
@@ -129,7 +134,7 @@ jobs:
           enable-cache: true
           version: "0.4.20"
           cache-dependency-glob: "requirements**.txt"
-          python-version: "3.12"
+          python-version: "3.11"
 
       - name: Setup node
         uses: actions/setup-node@v4
@@ -161,6 +166,12 @@ jobs:
             --tenant-id "$Env:AZURE_TENANT_ID"
         shell: pwsh
 
+      - name: Refresh azd environment variables
+        run: |
+            azd env refresh -e $AZURE_ENV_NAME --no-prompt
+        env:
+            AZD_INITIAL_ENVIRONMENT_CONFIG: ${{ secrets.AZD_INITIAL_ENVIRONMENT_CONFIG }}
+
       - name: Build frontend
         run: |
           cd ./app/frontend
@@ -173,37 +184,36 @@ jobs:
 
       - name: Run local server in background
         run: |
-          RUNNER_TRACKING_ID="" && (nohup python3 -m uvicorn fastapi_app:create_app --factory > serverlogs.out 2> serverlogs.err &)
+          cd app/backend
+          RUNNER_TRACKING_ID="" && (nohup python3 -m quart --app main:app run --port 50505 > serverlogs.out 2> serverlogs.err &)
+          cd ../..
 
       - name: Install evaluate dependencies
         run: |
           uv pip install -r evals/requirements.txt
 
-      - name: Run azd env get values command
-        run: |
-          azd env get-values
-
       - name: Evaluate local RAG flow
         run: |
-          python evals/evaluate.py --targeturl=http://127.0.0.1:8000/chat --resultsdir=evals/results/pr${{ github.event.issue.number }}
+          python evals/evaluate.py --targeturl=http://127.0.0.1:50505/chat --resultsdir=evals/results/pr${{ github.event.issue.number }}
+
+      - name: Upload eval results as build artifact
+        if: ${{ success() }}
+        uses: actions/upload-artifact@v4
+        with:
+            name: eval_result
+            path: ./evals/results/pr${{ github.event.issue.number }}
 
       - name: Upload server logs as build artifact
         uses: actions/upload-artifact@v4
         with:
           name: server_logs
-          path: ./serverlogs.out
+          path: ./app/backend/serverlogs.out
 
       - name: Upload server error logs as build artifact
         uses: actions/upload-artifact@v4
         with:
           name: server_error_logs
-          path: ./serverlogs.err
-
-      - name: Upload eval results as build artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: eval_result
-          path: ./evals/results/pr${{ github.event.issue.number }}
+          path: ./app/backend/serverlogs.err
 
       - name: Summarize results
         if: ${{ success() }}

app/backend/app.py

@@ -420,7 +420,7 @@ async def setup_clients():
     OPENAI_HOST = os.getenv("OPENAI_HOST", "azure")
     OPENAI_CHATGPT_MODEL = os.environ["AZURE_OPENAI_CHATGPT_MODEL"]
     OPENAI_EMB_MODEL = os.getenv("AZURE_OPENAI_EMB_MODEL_NAME", "text-embedding-ada-002")
-    OPENAI_EMB_DIMENSIONS = int(os.getenv("AZURE_OPENAI_EMB_DIMENSIONS", 1536))
+    OPENAI_EMB_DIMENSIONS = int(os.getenv("AZURE_OPENAI_EMB_DIMENSIONS") or 1536)
     # Used with Azure OpenAI deployments
     AZURE_OPENAI_SERVICE = os.getenv("AZURE_OPENAI_SERVICE")
     AZURE_OPENAI_GPT4V_DEPLOYMENT = os.environ.get("AZURE_OPENAI_GPT4V_DEPLOYMENT")
@@ -450,8 +450,8 @@ async def setup_clients():
     KB_FIELDS_CONTENT = os.getenv("KB_FIELDS_CONTENT", "content")
     KB_FIELDS_SOURCEPAGE = os.getenv("KB_FIELDS_SOURCEPAGE", "sourcepage")
 
-    AZURE_SEARCH_QUERY_LANGUAGE = os.getenv("AZURE_SEARCH_QUERY_LANGUAGE", "en-us")
-    AZURE_SEARCH_QUERY_SPELLER = os.getenv("AZURE_SEARCH_QUERY_SPELLER", "lexicon")
+    AZURE_SEARCH_QUERY_LANGUAGE = os.getenv("AZURE_SEARCH_QUERY_LANGUAGE") or "en-us"
+    AZURE_SEARCH_QUERY_SPELLER = os.getenv("AZURE_SEARCH_QUERY_SPELLER") or "lexicon"
     AZURE_SEARCH_SEMANTIC_RANKER = os.getenv("AZURE_SEARCH_SEMANTIC_RANKER", "free").lower()
 
     AZURE_SPEECH_SERVICE_ID = os.getenv("AZURE_SPEECH_SERVICE_ID")

app/backend/approaches/approach.py

@@ -152,7 +152,18 @@ async def search(
     ) -> List[Document]:
         search_text = query_text if use_text_search else ""
         search_vectors = vectors if use_vector_search else []
+        import logging
+
+        logger = logging.getLogger("scripts")
         if use_semantic_ranker:
+            logger.info("Using semantic ranker")
+            logger.info("Search text: %s", search_text)
+            logger.info("Filter: %s", filter)
+            logger.info("Top: %s", top)
+            logger.info("Query language: %s", self.query_language)
+            logger.info("Query speller: %s", self.query_speller)
+            logger.info("Semantic configuration: %s", "default")
+            logger.info("Semantic query: %s", query_text)
             results = await self.search_client.search(
                 search_text=search_text,
                 filter=filter,
@@ -166,6 +177,7 @@ async def search(
                 semantic_query=query_text,
             )
         else:
+            logger.info("Not using semantic ranker")
             results = await self.search_client.search(
                 search_text=search_text,
                 filter=filter,

azure.yaml

@@ -73,6 +73,12 @@ pipeline:
       - AZURE_OPENAI_EMB_DEPLOYMENT_VERSION
       - AZURE_OPENAI_EMB_DEPLOYMENT_SKU
       - AZURE_OPENAI_EMB_DIMENSIONS
+      - USE_EVAL
+      - AZURE_OPENAI_EVAL_MODEL
+      - AZURE_OPENAI_EVAL_MODEL_VERSION
+      - AZURE_OPENAI_EVAL_DEPLOYMENT
+      - AZURE_OPENAI_EVAL_DEPLOYMENT_SKU
+      - AZURE_OPENAI_EVAL_DEPLOYMENT_CAPACITY
       - AZURE_OPENAI_GPT4V_MODEL
       - AZURE_OPENAI_GPT4V_DEPLOYMENT
       - AZURE_OPENAI_GPT4V_DEPLOYMENT_CAPACITY