Skip to content

Commit 7eb1736

Browse files
author
Zhi Zhou
committed
List all sample code in readme and add shared_functions notebook
1 parent 51c22a8 commit 7eb1736

10 files changed

+663
-651
lines changed

Basic_Samples/GPT-4V/in_context_chatcompletions_example_restapi.ipynb renamed to Basic_Samples/GPT-4V/RAG_chatcompletions_example_restapi.ipynb

Lines changed: 46 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,15 @@
1313
},
1414
{
1515
"cell_type": "code",
16-
"execution_count": 1,
16+
"execution_count": 7,
1717
"id": "f4b3d21a",
1818
"metadata": {},
1919
"outputs": [],
2020
"source": [
2121
"import json\n",
2222
"import os\n",
23-
"import requests\n",
24-
"import base64"
23+
"import base64\n",
24+
"%run shared_functions.ipynb"
2525
]
2626
},
2727
{
@@ -32,12 +32,12 @@
3232
"### Setup Parameters\n",
3333
"\n",
3434
"\n",
35-
"Here we will load the configurations from _config.json_ file to setup search_service_endpoint, search_index_name, search_query_key, deployment_name, openai_api_base, openai_api_key and openai_api_version."
35+
"Here we will load the configurations from _config.json_ file to setup search_service_endpoint, search_index_name, and search_query_key"
3636
]
3737
},
3838
{
3939
"cell_type": "code",
40-
"execution_count": 2,
40+
"execution_count": 8,
4141
"id": "fd85fb30",
4242
"metadata": {},
4343
"outputs": [],
@@ -53,71 +53,58 @@
5353
"search_index_name = config_details['AZURE_SEARCH_INDEX_NAME']\n",
5454
"\n",
5555
"# Setting up the Azure Search service query key\n",
56-
"search_query_key = os.getenv(\"AZURE_SEARCH_QUERY_KEY\")\n",
57-
"\n",
58-
"# Setting up the deployment name\n",
59-
"deployment_name = config_details['GPT-4V_MODEL']\n",
60-
"\n",
61-
"# The base URL for your Azure OpenAI resource. e.g. \"https://<your resource name>.openai.azure.com\"\n",
62-
"openai_api_base = config_details['OPENAI_API_BASE']\n",
63-
"\n",
64-
"# The API key for your Azure OpenAI resource.\n",
65-
"openai_api_key = os.getenv(\"OPENAI_API_KEY\")\n",
66-
"\n",
67-
"# Currently OPENAI API have the following versions available: 2022-12-01. All versions follow the YYYY-MM-DD date structure.\n",
68-
"openai_api_version = config_details['OPENAI_API_VERSION']"
56+
"search_query_key = os.getenv(\"AZURE_SEARCH_QUERY_KEY\")"
57+
]
58+
},
59+
{
60+
"cell_type": "markdown",
61+
"metadata": {},
62+
"source": [
63+
"### Create Azure Search Index"
6964
]
7065
},
7166
{
7267
"cell_type": "code",
73-
"execution_count": 3,
68+
"execution_count": 10,
69+
"metadata": {},
70+
"outputs": [],
71+
"source": [
72+
"# Using the Azure Search service create the index with image embeddings\n",
73+
"#https://github.com/Azure/azure-search-vector-samples/blob/main/demo-python/code/azure-search-vector-image-index-creation-python-sample.ipynb"
74+
]
75+
},
76+
{
77+
"cell_type": "markdown",
78+
"metadata": {},
79+
"source": [
80+
"### Call GPT-4V API with Image"
81+
]
82+
},
83+
{
84+
"cell_type": "code",
85+
"execution_count": 9,
7486
"id": "b6165c63",
7587
"metadata": {},
7688
"outputs": [
7789
{
7890
"name": "stdout",
7991
"output_type": "stream",
8092
"text": [
81-
"The apple in the image appears to be a Gala apple.\n"
93+
"Gala\n"
8294
]
8395
}
8496
],
8597
"source": [
86-
"# Create Azure Search index (the link will be updated once it goes public)\n",
87-
"# https://github.com/zhizhoualan/cognitive-search-vector-pr/blob/main/demo-python/code/azure-search-vector-image-index-creation-python-sample.ipynb\n",
88-
"\n",
8998
"# System messages and user prompt\n",
9099
"sys_message = \"You are an AI assistant that helps people find information.\"\n",
91100
"user_prompt = \"What are the types of the apple(s) shown in this image?\"\n",
92101
"\n",
93102
"# Encode the image in base64\n",
94103
"image_file_path = \"../../common/images/test_Gala.jpeg\" # Another example including two apples: \"../../common/images/two_apples.jpeg\".\n",
95104
"with open(image_file_path, 'rb') as image_file:\n",
96-
" encoded_image = base64.b64encode(image_file.read()).decode('ascii')\n",
97-
" \n",
98-
"# Construct the API request URL\n",
99-
"api_url = f\"{openai_api_base}/openai/deployments/{deployment_name}/extensions/chat/completions?api-version={openai_api_version}\"\n",
100-
"\n",
101-
"# Including the api-key in HTTP headers\n",
102-
"headers = {\n",
103-
" \"Content-Type\": \"application/json\",\n",
104-
" \"api-key\": openai_api_key,\n",
105-
"}\n",
106-
"\n",
107-
"# Payload for the request\n",
108-
"payload = {\n",
109-
" \"model\": \"gpt-4-vision-preview\", \n",
110-
" \"dataSources\": [\n",
111-
" {\n",
112-
" \"type\": \"AzureCognitiveSearch\",\n",
113-
" \"parameters\": {\n",
114-
" \"endpoint\": search_service_endpoint,\n",
115-
" \"key\": search_query_key,\n",
116-
" \"indexName\": search_index_name\n",
117-
" }\n",
118-
" }\n",
119-
" ],\n",
120-
" \"messages\": [\n",
105+
" encoded_image = base64.b64encode(image_file.read()).decode('utf-8')\n",
106+
" \n",
107+
"messages = [\n",
121108
" {\n",
122109
" \"role\": \"system\",\n",
123110
" \"content\": [\n",
@@ -142,20 +129,19 @@
142129
" }\n",
143130
" ]\n",
144131
" }\n",
145-
" ],\n",
146-
" \"temperature\": 0.7,\n",
147-
" \"top_p\": 0.95,\n",
148-
" \"max_tokens\": 800\n",
149-
"}\n",
132+
" ]\n",
133+
"\n",
134+
"in_context_config = {\n",
135+
" 'endpoint': search_service_endpoint,\n",
136+
" 'key': search_query_key,\n",
137+
" 'indexName': search_index_name\n",
138+
"} \n",
150139
"\n",
151-
"# Send the request and handle the response\n",
152140
"try:\n",
153-
" response = requests.post(api_url, headers=headers, json=payload)\n",
154-
" response.raise_for_status() # Raise an error for bad HTTP status codes\n",
155-
" response_content = response.json()\n",
156-
" print(response_content['choices'][0]['message']['content'])\n",
157-
"except requests.RequestException as e:\n",
158-
" raise SystemExit(f\"Failed to make the request. Error: {e}\")"
141+
" response_content = call_GPT4V_image(messages, in_context=in_context_config)\n",
142+
" print(response_content['choices'][0]['message']['content']) # Print the content of the response\n",
143+
"except Exception as e:\n",
144+
" raise SystemExit(f\"Failed to call GPT-4V API. Error: {e}\")"
159145
]
160146
}
161147
],

‎Basic_Samples/GPT-4V/README.md‎

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,19 @@
11

22
# Introduction
33

4-
This repository contains samples demonstrating how to use GPT-4V for Chat Completions via REST API.
4+
This repository contains a collection of Jupyter notebooks demonstrating various use cases for interacting with the GPT-4V API, along with samples demonstrating how to use GPT-4V for Chat Completions via REST API. These examples provide practical guidance and accelerators for developers integrating GPT-4V functionalities in their applications.
5+
6+
## Contents
7+
| Notebook | Description | Type |
8+
|----------|-------------|-------|
9+
| [Basic Image in GPT-4V](basic_chatcompletions_example_restapi.ipynb) | Processing a single image input with GPT-4V. | Image |
10+
| [Handling Multiple Images in GPT-4V](mutiple_images_chatcompletions_example_restapi.ipynb) | Managing multiple image inputs in GPT-4V. | Image |
11+
| [Enhancing GPT-4V with RAG and Custom Data](RAG_chatcompletions_example_restapi.ipynb) | Enhancing capabilities by bringing custom data to augment image inputs in GPT-4V. | Image |
12+
| [Enhancing GPT-4V with Grounding Techniques](enhancement_grounding_chatcompletions_example_restapi.ipynb) | Applying grounding techniques to image inputs in GPT-4V. | Image |
13+
| [Enhancing GPT-4V with OCR Technique](enhancement_OCR_chatcompletions_example_restapi.ipynb) | Incorporating Optical Character Recognition (OCR) with image inputs in GPT-4V. | Image |
14+
| [Basic Video QnA in GPT-4V](video_chatcompletions_example_restapi.ipynb) | Conducting Q&A with video inputs in GPT-4V. | Video |
15+
| [Video Chunk Processing Sequentially in GPT-4V](video_chunk_chatcompletions_example_restapi.ipynb) | Sequential processing of video chunks in GPT-4V. | Video |
16+
517

618
## Installation
719
Install all Python modules and packages listed in the requirements.txt file using the below command.
@@ -11,7 +23,7 @@ pip install -r requirements.txt
1123
```
1224

1325
### Microsoft Azure Endpoints
14-
In order to use REST API with Microsoft Azure endpoints, you need to set GPT-4V_MODEL, OPENAI_API_BASE, OPENAI_API_VERSION & VISION_API_ENDPOINT in _config.json_ file.
26+
In order to use REST API with Microsoft Azure endpoints, you need to set a series of configurations such as GPT-4V_MODEL, OPENAI_API_BASE, OPENAI_API_VERSION & VISION_API_ENDPOINT in _config.json_ file.
1527

1628
```js
1729
{
@@ -20,6 +32,13 @@ In order to use REST API with Microsoft Azure endpoints, you need to set GPT-4V_
2032
"OPENAI_API_VERSION":"<OpenAI API Version>",
2133

2234
"VISION_API_ENDPOINT": "https://<Your Azure Vision Resource Name>.cognitiveservices.azure.com"
35+
36+
"AZURE_SEARCH_SERVICE_ENDPOINT": "https://<Your Azure Search Resource Name>.search.windows.net",
37+
"AZURE_SEARCH_INDEX_NAME": "<Your Azure Search Index Name>",
38+
39+
"VIDEO_SAS_URL": "<Your Azure Blob Storage SAS URL>",
40+
"VIDEO_INDEX_NAME": "<Your Azure Video Index Name>",
41+
"VIDEO_INDEX_ID": "<Your Azure Video Index ID>"
2342
}
2443
```
2544

@@ -46,7 +65,15 @@ Learn more about Azure OpenAI Service REST API [here](https://learn.microsoft.co
4665
Python 3.8+ <br>
4766
Jupyter Notebook 6.5.2
4867

49-
<br>
68+
69+
## Usage
70+
71+
Each notebook is self-contained and includes instructions specific to its scenario. Simply open a notebook in Jupyter and follow the steps outlined within it.
72+
73+
## Shared Functions
74+
75+
For convenience, commonly used functions across these notebooks are consolidated in [shared_functions.ipynb](shared_functions.ipynb). Import these functions in any notebook as needed.
76+
5077

5178
## Trademarks
5279

‎Basic_Samples/GPT-4V/basic_chatcompletions_example_restapi.ipynb‎

Lines changed: 23 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -11,57 +11,20 @@
1111
"# Chat Completions"
1212
]
1313
},
14-
{
15-
"cell_type": "code",
16-
"execution_count": 1,
17-
"id": "f4b3d21a",
18-
"metadata": {},
19-
"outputs": [],
20-
"source": [
21-
"import json\n",
22-
"import os\n",
23-
"import requests\n",
24-
"import base64"
25-
]
26-
},
27-
{
28-
"cell_type": "markdown",
29-
"id": "5b2d4a0f",
30-
"metadata": {},
31-
"source": [
32-
"### Setup Parameters\n",
33-
"\n",
34-
"\n",
35-
"Here we will load the configurations from _config.json_ file to setup deployment_name, openai_api_base, openai_api_key and openai_api_version."
36-
]
37-
},
3814
{
3915
"cell_type": "code",
4016
"execution_count": 2,
41-
"id": "fd85fb30",
17+
"id": "f4b3d21a",
4218
"metadata": {},
4319
"outputs": [],
4420
"source": [
45-
"# Load config values\n",
46-
"with open(r'config.json') as config_file:\n",
47-
" config_details = json.load(config_file)\n",
48-
" \n",
49-
"# Setting up the deployment name\n",
50-
"deployment_name = config_details['GPT-4V_MODEL']\n",
51-
"\n",
52-
"# The base URL for your Azure OpenAI resource. e.g. \"https://<your resource name>.openai.azure.com\"\n",
53-
"openai_api_base = config_details['OPENAI_API_BASE']\n",
54-
"\n",
55-
"# The API key for your Azure OpenAI resource.\n",
56-
"openai_api_key = os.getenv(\"OPENAI_API_KEY\")\n",
57-
"\n",
58-
"# Currently OPENAI API have the following versions available: 2022-12-01. All versions follow the YYYY-MM-DD date structure.\n",
59-
"openai_api_version = config_details['OPENAI_API_VERSION']"
21+
"import base64\n",
22+
"%run shared_functions.ipynb"
6023
]
6124
},
6225
{
6326
"cell_type": "code",
64-
"execution_count": 3,
27+
"execution_count": 18,
6528
"id": "aef62557",
6629
"metadata": {},
6730
"outputs": [
@@ -70,25 +33,20 @@
7033
"output_type": "stream",
7134
"text": [
7235
"1. Fresh produce\n",
73-
"2. Fruits and vegetables\n",
74-
"3. Colorful food\n",
75-
"4. Healthy eating\n",
76-
"5. Organic groceries\n",
77-
"6. Variety of produce\n",
78-
"7. Vegetarian ingredients\n",
79-
"8. Assorted vegetables\n",
80-
"9. Tropical fruits\n",
81-
"10. Farm fresh\n",
82-
"11. Nutrient-rich foods\n",
83-
"12. Vegan options\n",
84-
"13. Whole foods\n",
85-
"14. Raw vegetables\n",
86-
"15. Fruit assortment\n",
87-
"16. Market produce\n",
88-
"17. Dietary fiber sources\n",
89-
"18. Natural foods\n",
90-
"19. Agriculture harvest\n",
91-
"20. Culinary ingredients\n"
36+
"2. Assorted fruits\n",
37+
"3. Variety of vegetables\n",
38+
"4. Healthy food\n",
39+
"5. Colorful food assortment\n",
40+
"6. Organic produce\n",
41+
"7. Raw fruits and vegetables\n",
42+
"8. Vegan ingredients\n",
43+
"9. Nutritious diet\n",
44+
"10. Whole foods\n",
45+
"11. Farmers market\n",
46+
"12. Seasonal produce\n",
47+
"13. Plant-based diet\n",
48+
"14. Fresh harvest\n",
49+
"15. Food background\n"
9250
]
9351
}
9452
],
@@ -109,19 +67,7 @@
10967
"with open(image_file_path, 'rb') as image_file:\n",
11068
" encoded_image = base64.b64encode(image_file.read()).decode('utf-8')\n",
11169
"\n",
112-
"# Construct the API request URL\n",
113-
"api_url = f\"{openai_api_base}/openai/deployments/{deployment_name}/chat/completions?api-version={openai_api_version}\"\n",
114-
"\n",
115-
"# Including the api-key in HTTP headers\n",
116-
"headers = {\n",
117-
" \"Content-Type\": \"application/json\",\n",
118-
" \"api-key\": openai_api_key,\n",
119-
"}\n",
120-
"\n",
121-
"# Payload for the request\n",
122-
"payload = {\n",
123-
" \"model\": \"gpt-4-vision-preview\",\n",
124-
" \"messages\": [\n",
70+
"messages = [\n",
12571
" {\n",
12672
" \"role\": \"system\",\n",
12773
" \"content\": [\n",
@@ -146,20 +92,13 @@
14692
" }\n",
14793
" ]\n",
14894
" }\n",
149-
" ],\n",
150-
" \"temperature\": 0.7,\n",
151-
" \"top_p\": 0.95,\n",
152-
" \"max_tokens\": 800\n",
153-
"}\n",
95+
" ]\n",
15496
"\n",
155-
"# Send the request and handle the response\n",
15697
"try:\n",
157-
" response = requests.post(api_url, headers=headers, json=payload)\n",
158-
" response.raise_for_status() # Raise an error for bad HTTP status codes\n",
159-
" response_content = response.json()\n",
98+
" response_content = call_GPT4V_image(messages)\n",
16099
" print(response_content['choices'][0]['message']['content']) # Print the content of the response\n",
161-
"except requests.RequestException as e:\n",
162-
" raise SystemExit(f\"Failed to make the request. Error: {e}\")"
100+
"except Exception as e:\n",
101+
" raise SystemExit(f\"Failed to call GPT-4V API. Error: {e}\")"
163102
]
164103
}
165104
],

0 commit comments

Comments
 (0)