Merge branch 'aws-samples:main' into main

Author: giuseppe-zappia

workshops/diy-agents-with-sagemaker-and-bedrock/0-setup/2-setup-sagemaker-endpoint.ipynb

@@ -26,8 +26,36 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "fd08268e",
-   "metadata": {},
-   "outputs": [],
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-10-15T17:49:49.105299Z",
+     "iopub.status.busy": "2025-10-15T17:49:49.105048Z",
+     "iopub.status.idle": "2025-10-15T18:01:24.859971Z",
+     "shell.execute_reply": "2025-10-15T18:01:24.859446Z",
+     "shell.execute_reply.started": "2025-10-15T17:49:49.105277Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n",
+      "sagemaker.config INFO - Not applying SDK defaults from location: /home/sagemaker-user/.config/sagemaker/config.yaml\n",
+      "----------------------!"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "<sagemaker.djl_inference.djl_predictor.DJLPredictor at 0x7f5e46e66d50>"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "from sagemaker.djl_inference import DJLModel\n",
     "from sagemaker.utils import name_from_base\n",
@@ -44,18 +72,20 @@
     "endpoint_name = name_from_base(model_id.split(\"/\")[1]+\"-ep\")\n",
     "model = DJLModel(\n",
     "    name=model_name, role=role,\n",
-    "    image_uri=f\"763104351884.dkr.ecr.{boto3.Session().region_name}.amazonaws.com/djl-inference:0.33.0-lmi15.0.0-cu128-v1.3\",\n",
+    "    image_uri=f\"763104351884.dkr.ecr.{boto3.Session().region_name}.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128-v1.2\",\n",
     "    env={\n",
-    "        \"HF_MODEL_ID\": model_id, # config: https://qwen.readthedocs.io/en/latest/framework/function_call.html#vllm \n",
-    "        \"OPTION_MAX_MODEL_LEN\": f\"{1024*20}\",\n",
+    "        \"HF_MODEL_ID\": model_id,\n",
+    "        \"OPTION_MAX_MODEL_LEN\": f\"{1024*16}\",\n",
     "        \"OPTION_QUANTIZE\": \"fp8\",\n",
-    "        # vllm serve {model_id} --enable-auto-tool-choice --tool-call-parser hermes\n",
-    "        \"OPTION_ROLLING_BATCH\": \"vllm\",\n",
+    "        'OPTION_DTYPE': 'bf16',\n",
+    "        'SERVING_FAIL_FAST': 'true',\n",
+    "        'OPTION_ROLLING_BATCH': 'disable',\n",
+    "        'OPTION_ASYNC_MODE': 'true',\n",
+    "        'OPTION_ENTRYPOINT': 'djl_python.lmi_vllm.vllm_async_service',\n",
     "        \"OPTION_ENABLE_AUTO_TOOL_CHOICE\": \"true\",\n",
     "        \"OPTION_TOOL_CALL_PARSER\": \"hermes\",\n",
-    "        # --enable-reasoning --reasoning-parser deepseek_r1\n",
-    "        # \"OPTION_ENABLE_REASONING\": \"true\",\n",
-    "        # \"OPTION_REASONING_PARSER\": \"qwen3\", # currently not available in djl lmi15\n",
+    "        \"OPTION_ENABLE_REASONING\": \"true\",\n",
+    "        \"OPTION_REASONING_PARSER\": \"qwen3\",\n",
     "    }\n",
     ")\n",
     "model.deploy(\n",
@@ -67,10 +97,27 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "id": "18e6de49-11f7-4e36-b7bb-322282a51e53",
-   "metadata": {},
-   "outputs": [],
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-10-15T18:01:24.860899Z",
+     "iopub.status.busy": "2025-10-15T18:01:24.860666Z",
+     "iopub.status.idle": "2025-10-15T18:01:24.864407Z",
+     "shell.execute_reply": "2025-10-15T18:01:24.863955Z",
+     "shell.execute_reply.started": "2025-10-15T18:01:24.860877Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Endpoint name: Qwen3-4B-ep-2025-10-15-17-49-51-128\n",
+      "Stored 'SAGEMAKER_ENDPOINT_NAME' (str)\n"
+     ]
+    }
+   ],
    "source": [
     "SAGEMAKER_ENDPOINT_NAME = model.endpoint_name\n",
     "print(f\"Endpoint name: {SAGEMAKER_ENDPOINT_NAME}\")\n",