Merge branch 'f/sglang-support' of https://github.com/VectorInstitute/vector-inference into f/sglang-support

Author: XkunW

.github/workflows/docker.yml

@@ -13,6 +13,7 @@ on:
   pull_request:
     branches:
       - main
+      - f/sglang-support
     paths:
       - Dockerfile
       - .github/workflows/docker.yml
@@ -22,8 +23,7 @@ jobs:
   push_to_registry:
     name: Push Docker image to Docker Hub
     runs-on:
-      - self-hosted
-      - docker
+      - ubuntu-latest
     steps:
       - name: Checkout repository
         uses: actions/checkout@v5.0.0
@@ -34,6 +34,24 @@ jobs:
           VERSION=$(grep -A 1 'name = "vllm"' uv.lock | grep version | cut -d '"' -f 2)
           echo "version=$VERSION" >> $GITHUB_OUTPUT
 
+      - name: Maximize build space
+        run: |
+          echo "Disk space before cleanup:"
+          df -h
+          # Remove unnecessary pre-installed software
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo rm -rf /usr/local/share/boost
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          # Clean apt cache
+          sudo apt-get clean
+          # Remove docker images
+          docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
+          echo "Disk space after cleanup:"
+          df -h
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 

.pre-commit-config.yaml

@@ -17,7 +17,7 @@ repos:
     - id: check-toml
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.14.4'
+    rev: 'v0.14.5'
     hooks:
     - id: ruff
       args: [--fix, --exit-non-zero-on-fix]

Dockerfile

@@ -57,7 +57,9 @@ WORKDIR /vec-inf
 COPY . /vec-inf
 
 # Install project dependencies with build requirements
-RUN uv pip install --system -e .[dev] --prerelease=allow
+# Use --no-cache to prevent uv from storing both downloaded and extracted packages
+RUN uv pip install --system -e .[dev] --prerelease=allow --no-cache && \
+    rm -rf /root/.cache/uv /tmp/*
 
 # Install a single, system NCCL (from NVIDIA CUDA repo in base image)
 RUN apt-get update && apt-get install -y --allow-change-held-packages\

vec_inf/config/models.yaml

@@ -1059,12 +1059,24 @@ models:
     model_family: gpt-oss
     model_variant: 120b
     model_type: LLM
-    gpus_per_node: 4
-    num_nodes: 2
+    gpus_per_node: 2
+    num_nodes: 1
     vocab_size: 201088
     time: 08:00:00
     resource_type: l40s
     vllm_args:
+      --tensor-parallel-size: 2
+      --max-model-len: 32768
+  Llama-4-Maverick-17B-128E-Instruct:
+    model_family: Llama-4
+    model_variant: Maverick-17B-128E-Instruct
+    model_type: VLM
+    gpus_per_node: 4
+    num_nodes: 8
+    vocab_size: 202048
+    time: 03:00:00
+    resource_type: l40s
+    vllm_args:
+      --max-model-len: 16384
       --tensor-parallel-size: 4
-      --pipeline-parallel-size: 2
-      --max-model-len: 40960
+      --pipeline-parallel-size: 8

vec_inf/find_port.sh

@@ -28,7 +28,16 @@ find_available_port() {
     local base_port=$2
     local max_port=$3
 
-    for ((port=base_port; port<=max_port; port++)); do
+    # Generate shuffled list of ports; fallback to sequential if shuf not present
+    if command -v shuf >/dev/null 2>&1; then
+        local port_list
+        port_list=$(shuf -i "${base_port}-${max_port}")
+    else
+        local port_list
+        port_list=$(seq $base_port $max_port)
+    fi
+
+    for port in $port_list; do
         if is_port_available $ip $port; then
             echo $port
             return