From ce552e297f8285f7f02eb094661cc25e8b34b30d Mon Sep 17 00:00:00 2001 From: willhath-openai Date: Wed, 14 May 2025 17:27:41 -0700 Subject: [PATCH 1/3] wip --- .../use-cases/responses-evaluation.ipynb | 151 ++++++++++++++++++ 1 file changed, 151 insertions(+) diff --git a/examples/evaluation/use-cases/responses-evaluation.ipynb b/examples/evaluation/use-cases/responses-evaluation.ipynb index 278f635dbb..4e7391e4cf 100644 --- a/examples/evaluation/use-cases/responses-evaluation.ipynb +++ b/examples/evaluation/use-cases/responses-evaluation.ipynb @@ -16,14 +16,21 @@ }, { "cell_type": "code", +<<<<<<< HEAD "execution_count": 30, +======= + "execution_count": 68, +>>>>>>> 562e852 (wip) "metadata": {}, "outputs": [], "source": [ "import openai\n", "import os\n", "\n", +<<<<<<< HEAD "\n", +======= +>>>>>>> 562e852 (wip) "client = openai.OpenAI()" ] }, @@ -44,7 +51,11 @@ "source": [ "openai_sdk_file_path = os.path.dirname(openai.__file__)\n", "\n", +<<<<<<< HEAD "# Get some example code files from the OpenAI SDK \n", +======= + "# Get some example code files from the OpenAI SDK\n", +>>>>>>> 562e852 (wip) "file_paths = [\n", " os.path.join(openai_sdk_file_path, \"resources\", \"evals\", \"evals.py\"),\n", " os.path.join(openai_sdk_file_path, \"resources\", \"responses\", \"responses.py\"),\n", @@ -60,12 +71,20 @@ "cell_type": "markdown", "metadata": {}, "source": [ +<<<<<<< HEAD "Now, lets generate some responses. " +======= + "Now, lets generate some responses" +>>>>>>> 562e852 (wip) ] }, { "cell_type": "code", +<<<<<<< HEAD "execution_count": null, +======= + "execution_count": 66, +>>>>>>> 562e852 (wip) "metadata": {}, "outputs": [], "source": [ @@ -86,6 +105,7 @@ " ],\n", " model=\"gpt-4o-mini\",\n", " )\n", +<<<<<<< HEAD " print(response.output_text)" ] }, @@ -94,15 +114,24 @@ "metadata": {}, "source": [ "Note that in order for this to work, you'll have to be doing this on an org where data logging isn't disabled (through zdr, etc). If you aren't sure if this is the case for you, go to https://platform.openai.com/logs?api=responses and see if you can see the responses you just generated." +======= + " break\n", + " print(response.id)" +>>>>>>> 562e852 (wip) ] }, { "cell_type": "code", +<<<<<<< HEAD "execution_count": 31, +======= + "execution_count": 49, +>>>>>>> 562e852 (wip) "metadata": {}, "outputs": [], "source": [ "grader_system_prompt = \"\"\"\n", +<<<<<<< HEAD "You are **Code-Explanation Grader**, an expert software engineer and technical writer. \n", "Your job is to score how well *Model A* explained the purpose and behaviour of a given source-code file.\n", "\n", @@ -162,6 +191,66 @@ "Then set `\"result\": \"4.0\"` (example).\n", "\n", "Be rigorous and unbiased.\n", +======= + "We've created a consumer-facing Evals product to help AI integrators quickly and clearly understand their models' real-world performance. Your role is to serve as a Universal Evaluator, automatically grading responses to measure how well each model output addresses user needs and expectations.\n", + "\n", + "Given the conversation messages, assign a quality score in the `result` key of the response in the inclusive range between 1.0 (poor) and 7.0 (excellent). Customers will analyze your collective scores and reasoning to gain actionable insights into their models' performance.\n", + "\n", + "---\n", + "\n", + "## Things to Consider\n", + "\n", + "- Evaluate the overall value provided to the user\n", + "- Verify all claims and do not take the AI's statements at face value! Errors might be very hard to find and well hidden.\n", + "- Differentiate between minor errors (slight utility reduction) and major errors (significant trust or safety impact).\n", + "- Reward answers that closely follow user instructions.\n", + "- Reserve the highest and lowest reward scores for cases where you have complete certainty about correctness and utility.\n", + "\n", + "\n", + "---\n", + "\n", + "## Secondary Labels to Support Final Utility Score Prediction\n", + "\n", + "To help you assign an accurate final utility score, first analyze and predict several important aspects of the AI response. Crucially, these intermediate evaluations should precede your final utility score prediction.\n", + "\n", + "Your structured output must match the provided schema:\n", + "\n", + "- `steps`: A JSON array of objects, each containing:\n", + " - `description`: A detailed explanation of your reasoning for each step.\n", + " - `result`: The float score reached based on the reasoning in this step.\n", + "\n", + "### Steps to Predict (in order):\n", + "\n", + "1. **major_errors**\n", + " - *description*: Identify and explain any significant errors.\n", + " - *conclusion*: List major errors found, or indicate \"None\".\n", + "\n", + "2. **minor_errors**\n", + " - *description*: Identify and explain any minor inaccuracies.\n", + " - *conclusion*: List minor errors found, or indicate \"None\".\n", + "\n", + "3. **potential_improvements**\n", + " - *description*: Suggest enhancements that would improve the response.\n", + " - *conclusion*: List suggested improvements, or indicate \"None\".\n", + "\n", + "---\n", + "\n", + "## JSON Response Structure\n", + "\n", + "Once you predicted all the above fields you need to assign a float between 1 and 7 to indicate the response's utility compared to the alternative responses. Use your best judgment for the meaning of `final_score`. Your response should be a JSON that can be loaded with json.loads in Python and contains:\n", + "- steps: An array of objects representing your reasoning steps. Each step includes:\n", + " - description (string): Detailed reasoning for this step.\n", + " - result (string): The float score derived from this reasoning.\n", + "- result (float): A numeric quality score as a string, in the inclusive range [1,7].\n", + "\n", + "---\n", + "\n", + "## Notes\n", + "\n", + "- Be meticulous in identifying errors, especially subtle or high-impact ones.\n", + "- Avoid being too kind by giving overly high scores easily, it's important to often keep a gap at the top to continue having signal for improvement. Only use [6.5, 7) if the answer is truly mind blowing and you don't see how it could have been improved.\n", + "- Never take the AI's responses at face value—verify everything thoroughly.\n", +>>>>>>> 562e852 (wip) "\"\"\"\n", "user_input_message = \"\"\"**User input**\n", "\n", @@ -169,13 +258,21 @@ "\n", "**Response to evaluate**\n", "\n", +<<<<<<< HEAD "{{sample.output_text}}\n", +======= + "{{sample.output_text}\n", +>>>>>>> 562e852 (wip) "\"\"\"" ] }, { "cell_type": "code", +<<<<<<< HEAD "execution_count": 25, +======= + "execution_count": 71, +>>>>>>> 562e852 (wip) "metadata": {}, "outputs": [], "source": [ @@ -197,8 +294,12 @@ " \"content\": user_input_message,\n", " },\n", " ],\n", +<<<<<<< HEAD " \"range\": [1, 7],\n", " \"pass_threshold\": 5.5,\n", +======= + " \"pass_threshold\": 0.5\n", +>>>>>>> 562e852 (wip) " }\n", " ]\n", ")" @@ -213,6 +314,7 @@ }, { "cell_type": "code", +<<<<<<< HEAD "execution_count": 26, "metadata": {}, "outputs": [], @@ -223,6 +325,17 @@ " data_source={\n", " \"type\": \"responses\",\n", " \"source\": {\"type\": \"responses\", \"limit\": len(file_paths)}, # just grab the most recent responses\n", +======= + "execution_count": 72, + "metadata": {}, + "outputs": [], + "source": [ + "gpt_4o_run = client.evals.runs.create(\n", + " eval_id=logs_eval.id,\n", + " data_source={\n", + " \"type\": \"responses\",\n", + " \"source\": {\"type\": \"responses\", \"limit\": len(file_paths)},\n", +>>>>>>> 562e852 (wip) " },\n", ")" ] @@ -236,12 +349,20 @@ }, { "cell_type": "code", +<<<<<<< HEAD "execution_count": 27, "metadata": {}, "outputs": [], "source": [ "gpt_41_mini_run = client.evals.runs.create(\n", " name=\"gpt-4.1-mini\",\n", +======= + "execution_count": 73, + "metadata": {}, + "outputs": [], + "source": [ + "gpt_41_run = client.evals.runs.create(\n", +>>>>>>> 562e852 (wip) " eval_id=logs_eval.id,\n", " data_source={\n", " \"type\": \"responses\",\n", @@ -264,6 +385,7 @@ }, { "cell_type": "code", +<<<<<<< HEAD "execution_count": null, "metadata": {}, "outputs": [], @@ -274,12 +396,41 @@ { "cell_type": "markdown", "metadata": {}, +======= + "execution_count": 74, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'https://platform.openai.com/evaluations/eval_6822dd1c4d308190855601dc7fe982bd?project_id=proj_YFvXZA3VxHbZx5aSJv6SAM5B&run_id=evalrun_6822dd1f89b88190a3287c706fd599f4'" + ] + }, + "execution_count": 74, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "gpt_41_run.report_url" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], +>>>>>>> 562e852 (wip) "source": [] } ], "metadata": { "kernelspec": { +<<<<<<< HEAD "display_name": "Python 3", +======= + "display_name": "openai", +>>>>>>> 562e852 (wip) "language": "python", "name": "python3" }, From d80ed1fc8d87c14584fc17981cebd9509ed138a1 Mon Sep 17 00:00:00 2001 From: willhath-openai Date: Wed, 14 May 2025 17:29:29 -0700 Subject: [PATCH 2/3] wip --- registry.yaml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/registry.yaml b/registry.yaml index de54e08bf1..ff81e2d418 100644 --- a/registry.yaml +++ b/registry.yaml @@ -4,7 +4,6 @@ # should build pages for, and indicates metadata such as tags, creation date and # authors for each page. - - title: Better performance from reasoning models using the Responses API path: examples/responses_api/reasoning_items.ipynb date: 2025-05-11 @@ -52,40 +51,40 @@ - legal - pharma -- title: EvalsAPI Use-case - Detecting prompt regressions +- title: Evals API Use-case - Detecting prompt regressions path: examples/evaluation/use-cases/regression.ipynb date: 2025-04-08 authors: - josiah-openai tags: - - evalsapi + - evals-api - completions -- title: EvalsAPI Use-case - Bulk model and prompt experimentation +- title: Evals API Use-case - Bulk model and prompt experimentation path: examples/evaluation/use-cases/bulk-experimentation.ipynb date: 2025-04-08 authors: - josiah-openai tags: - - evalsapi + - evals-api - completions -- title: EvalsAPI Use-case - Monitoring stored completions +- title: Evals API Use-case - Monitoring stored completions path: examples/evaluation/use-cases/completion-monitoring.ipynb date: 2025-04-08 authors: - josiah-openai tags: - - evalsapi + - evals-api - completions -- title: EvalsAPI Use-case - Responses Evaluation +- title: Evals API Use-case - Responses Evaluation path: examples/evaluation/use-cases/responses-evaluation.ipynb date: 2025-05-13 authors: - willhath-openai tags: - - evalsapi + - evals-api - responses - title: Multi-Tool Orchestration with RAG approach using OpenAI's Responses API From f50cbbc07104f1312fc007b0192b9f6ba69aeab8 Mon Sep 17 00:00:00 2001 From: willhath-openai Date: Thu, 15 May 2025 16:27:29 -0700 Subject: [PATCH 3/3] Revert "wip" This reverts commit ce552e297f8285f7f02eb094661cc25e8b34b30d. --- .../use-cases/responses-evaluation.ipynb | 151 ------------------ 1 file changed, 151 deletions(-) diff --git a/examples/evaluation/use-cases/responses-evaluation.ipynb b/examples/evaluation/use-cases/responses-evaluation.ipynb index 4e7391e4cf..278f635dbb 100644 --- a/examples/evaluation/use-cases/responses-evaluation.ipynb +++ b/examples/evaluation/use-cases/responses-evaluation.ipynb @@ -16,21 +16,14 @@ }, { "cell_type": "code", -<<<<<<< HEAD "execution_count": 30, -======= - "execution_count": 68, ->>>>>>> 562e852 (wip) "metadata": {}, "outputs": [], "source": [ "import openai\n", "import os\n", "\n", -<<<<<<< HEAD "\n", -======= ->>>>>>> 562e852 (wip) "client = openai.OpenAI()" ] }, @@ -51,11 +44,7 @@ "source": [ "openai_sdk_file_path = os.path.dirname(openai.__file__)\n", "\n", -<<<<<<< HEAD "# Get some example code files from the OpenAI SDK \n", -======= - "# Get some example code files from the OpenAI SDK\n", ->>>>>>> 562e852 (wip) "file_paths = [\n", " os.path.join(openai_sdk_file_path, \"resources\", \"evals\", \"evals.py\"),\n", " os.path.join(openai_sdk_file_path, \"resources\", \"responses\", \"responses.py\"),\n", @@ -71,20 +60,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ -<<<<<<< HEAD "Now, lets generate some responses. " -======= - "Now, lets generate some responses" ->>>>>>> 562e852 (wip) ] }, { "cell_type": "code", -<<<<<<< HEAD "execution_count": null, -======= - "execution_count": 66, ->>>>>>> 562e852 (wip) "metadata": {}, "outputs": [], "source": [ @@ -105,7 +86,6 @@ " ],\n", " model=\"gpt-4o-mini\",\n", " )\n", -<<<<<<< HEAD " print(response.output_text)" ] }, @@ -114,24 +94,15 @@ "metadata": {}, "source": [ "Note that in order for this to work, you'll have to be doing this on an org where data logging isn't disabled (through zdr, etc). If you aren't sure if this is the case for you, go to https://platform.openai.com/logs?api=responses and see if you can see the responses you just generated." -======= - " break\n", - " print(response.id)" ->>>>>>> 562e852 (wip) ] }, { "cell_type": "code", -<<<<<<< HEAD "execution_count": 31, -======= - "execution_count": 49, ->>>>>>> 562e852 (wip) "metadata": {}, "outputs": [], "source": [ "grader_system_prompt = \"\"\"\n", -<<<<<<< HEAD "You are **Code-Explanation Grader**, an expert software engineer and technical writer. \n", "Your job is to score how well *Model A* explained the purpose and behaviour of a given source-code file.\n", "\n", @@ -191,66 +162,6 @@ "Then set `\"result\": \"4.0\"` (example).\n", "\n", "Be rigorous and unbiased.\n", -======= - "We've created a consumer-facing Evals product to help AI integrators quickly and clearly understand their models' real-world performance. Your role is to serve as a Universal Evaluator, automatically grading responses to measure how well each model output addresses user needs and expectations.\n", - "\n", - "Given the conversation messages, assign a quality score in the `result` key of the response in the inclusive range between 1.0 (poor) and 7.0 (excellent). Customers will analyze your collective scores and reasoning to gain actionable insights into their models' performance.\n", - "\n", - "---\n", - "\n", - "## Things to Consider\n", - "\n", - "- Evaluate the overall value provided to the user\n", - "- Verify all claims and do not take the AI's statements at face value! Errors might be very hard to find and well hidden.\n", - "- Differentiate between minor errors (slight utility reduction) and major errors (significant trust or safety impact).\n", - "- Reward answers that closely follow user instructions.\n", - "- Reserve the highest and lowest reward scores for cases where you have complete certainty about correctness and utility.\n", - "\n", - "\n", - "---\n", - "\n", - "## Secondary Labels to Support Final Utility Score Prediction\n", - "\n", - "To help you assign an accurate final utility score, first analyze and predict several important aspects of the AI response. Crucially, these intermediate evaluations should precede your final utility score prediction.\n", - "\n", - "Your structured output must match the provided schema:\n", - "\n", - "- `steps`: A JSON array of objects, each containing:\n", - " - `description`: A detailed explanation of your reasoning for each step.\n", - " - `result`: The float score reached based on the reasoning in this step.\n", - "\n", - "### Steps to Predict (in order):\n", - "\n", - "1. **major_errors**\n", - " - *description*: Identify and explain any significant errors.\n", - " - *conclusion*: List major errors found, or indicate \"None\".\n", - "\n", - "2. **minor_errors**\n", - " - *description*: Identify and explain any minor inaccuracies.\n", - " - *conclusion*: List minor errors found, or indicate \"None\".\n", - "\n", - "3. **potential_improvements**\n", - " - *description*: Suggest enhancements that would improve the response.\n", - " - *conclusion*: List suggested improvements, or indicate \"None\".\n", - "\n", - "---\n", - "\n", - "## JSON Response Structure\n", - "\n", - "Once you predicted all the above fields you need to assign a float between 1 and 7 to indicate the response's utility compared to the alternative responses. Use your best judgment for the meaning of `final_score`. Your response should be a JSON that can be loaded with json.loads in Python and contains:\n", - "- steps: An array of objects representing your reasoning steps. Each step includes:\n", - " - description (string): Detailed reasoning for this step.\n", - " - result (string): The float score derived from this reasoning.\n", - "- result (float): A numeric quality score as a string, in the inclusive range [1,7].\n", - "\n", - "---\n", - "\n", - "## Notes\n", - "\n", - "- Be meticulous in identifying errors, especially subtle or high-impact ones.\n", - "- Avoid being too kind by giving overly high scores easily, it's important to often keep a gap at the top to continue having signal for improvement. Only use [6.5, 7) if the answer is truly mind blowing and you don't see how it could have been improved.\n", - "- Never take the AI's responses at face value—verify everything thoroughly.\n", ->>>>>>> 562e852 (wip) "\"\"\"\n", "user_input_message = \"\"\"**User input**\n", "\n", @@ -258,21 +169,13 @@ "\n", "**Response to evaluate**\n", "\n", -<<<<<<< HEAD "{{sample.output_text}}\n", -======= - "{{sample.output_text}\n", ->>>>>>> 562e852 (wip) "\"\"\"" ] }, { "cell_type": "code", -<<<<<<< HEAD "execution_count": 25, -======= - "execution_count": 71, ->>>>>>> 562e852 (wip) "metadata": {}, "outputs": [], "source": [ @@ -294,12 +197,8 @@ " \"content\": user_input_message,\n", " },\n", " ],\n", -<<<<<<< HEAD " \"range\": [1, 7],\n", " \"pass_threshold\": 5.5,\n", -======= - " \"pass_threshold\": 0.5\n", ->>>>>>> 562e852 (wip) " }\n", " ]\n", ")" @@ -314,7 +213,6 @@ }, { "cell_type": "code", -<<<<<<< HEAD "execution_count": 26, "metadata": {}, "outputs": [], @@ -325,17 +223,6 @@ " data_source={\n", " \"type\": \"responses\",\n", " \"source\": {\"type\": \"responses\", \"limit\": len(file_paths)}, # just grab the most recent responses\n", -======= - "execution_count": 72, - "metadata": {}, - "outputs": [], - "source": [ - "gpt_4o_run = client.evals.runs.create(\n", - " eval_id=logs_eval.id,\n", - " data_source={\n", - " \"type\": \"responses\",\n", - " \"source\": {\"type\": \"responses\", \"limit\": len(file_paths)},\n", ->>>>>>> 562e852 (wip) " },\n", ")" ] @@ -349,20 +236,12 @@ }, { "cell_type": "code", -<<<<<<< HEAD "execution_count": 27, "metadata": {}, "outputs": [], "source": [ "gpt_41_mini_run = client.evals.runs.create(\n", " name=\"gpt-4.1-mini\",\n", -======= - "execution_count": 73, - "metadata": {}, - "outputs": [], - "source": [ - "gpt_41_run = client.evals.runs.create(\n", ->>>>>>> 562e852 (wip) " eval_id=logs_eval.id,\n", " data_source={\n", " \"type\": \"responses\",\n", @@ -385,7 +264,6 @@ }, { "cell_type": "code", -<<<<<<< HEAD "execution_count": null, "metadata": {}, "outputs": [], @@ -396,41 +274,12 @@ { "cell_type": "markdown", "metadata": {}, -======= - "execution_count": 74, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'https://platform.openai.com/evaluations/eval_6822dd1c4d308190855601dc7fe982bd?project_id=proj_YFvXZA3VxHbZx5aSJv6SAM5B&run_id=evalrun_6822dd1f89b88190a3287c706fd599f4'" - ] - }, - "execution_count": 74, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "gpt_41_run.report_url" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], ->>>>>>> 562e852 (wip) "source": [] } ], "metadata": { "kernelspec": { -<<<<<<< HEAD "display_name": "Python 3", -======= - "display_name": "openai", ->>>>>>> 562e852 (wip) "language": "python", "name": "python3" },