Upgrade HF Image URI 2.0.2 & Least Outstanding Requests Strategy

Author: windson

llama-2-hf-tgi/llama-2-13b-chat-hf/1-deploy-llama-2-13b-chat-hf-tgi-sagemaker.ipynb

@@ -135,7 +135,7 @@
     "# retrieve the llm image uri\n",
     "llm_image = get_huggingface_llm_image_uri(\n",
     "  \"huggingface\",\n",
-    "  version=\"1.0.3\"\n",
+    "  version=\"2.0.2\"\n",
     ")\n",
     "\n",
     "# print ecr image uri\n",
@@ -244,6 +244,9 @@
     "    initial_instance_count=1,\n",
     "    instance_type=instance_type,\n",
     "    container_startup_health_check_timeout=health_check_timeout,\n",
+    "    routing_config = {\n",
+    "        \"RoutingStrategy\":  sagemaker.enums.RoutingStrategy.LEAST_OUTSTANDING_REQUESTS\n",
+    "    }\n",
     ")"
    ]
   },

llama-2-hf-tgi/llama-2-70b-chat-hf/1-deploy-llama-2-70b-chat-hf-tgi-sagemaker.ipynb

@@ -135,7 +135,7 @@
     "# retrieve the llm image uri\n",
     "llm_image = get_huggingface_llm_image_uri(\n",
     "  \"huggingface\",\n",
-    "  version=\"1.0.3\"\n",
+    "  version=\"2.0.2\"\n",
     ")\n",
     "\n",
     "# print ecr image uri\n",
@@ -246,6 +246,9 @@
     "    initial_instance_count=1,\n",
     "    instance_type=instance_type,\n",
     "    container_startup_health_check_timeout=health_check_timeout,\n",
+    "    routing_config = {\n",
+    "        \"RoutingStrategy\":  sagemaker.enums.RoutingStrategy.LEAST_OUTSTANDING_REQUESTS\n",
+    "    }\n",
     ")"
    ]
   },

llama-2-hf-tgi/llama-2-7b-chat-hf/1-deploy-llama-2-7b-chat-hf-tgi-sagemaker.ipynb

@@ -135,7 +135,7 @@
     "# retrieve the llm image uri\n",
     "llm_image = get_huggingface_llm_image_uri(\n",
     "  \"huggingface\",\n",
-    "  version=\"1.0.3\"\n",
+    "  version=\"2.0.2\"\n",
     ")\n",
     "\n",
     "# print ecr image uri\n",
@@ -244,6 +244,9 @@
     "    initial_instance_count=1,\n",
     "    instance_type=instance_type,\n",
     "    container_startup_health_check_timeout=health_check_timeout,\n",
+    "    routing_config = {\n",
+    "        \"RoutingStrategy\":  sagemaker.enums.RoutingStrategy.LEAST_OUTSTANDING_REQUESTS\n",
+    "    }\n",
     ")"
    ]
   },