Split vllm and sglang dependencies into 2 groups and 2 docker images

Author: XkunW

.github/workflows/docker.yml

@@ -7,31 +7,36 @@ on:
     branches:
       - main
     paths:
-      - Dockerfile
+      - vllm.Dockerfile
+      - sglang.Dockerfile
       - .github/workflows/docker.yml
       - uv.lock
   pull_request:
     branches:
       - main
       - f/sglang-support
     paths:
-      - Dockerfile
+      - vllm.Dockerfile
+      - sglang.Dockerfile
       - .github/workflows/docker.yml
       - uv.lock
 
 jobs:
   push_to_registry:
-    name: Push Docker image to Docker Hub
+    name: Build and push Docker images
     runs-on:
       - ubuntu-latest
+    strategy:
+      matrix:
+        backend: [vllm, sglang]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v5.0.0
 
-      - name: Extract vLLM version
-        id: vllm-version
+      - name: Extract backend version
+        id: backend-version
         run: |
-          VERSION=$(grep -A 1 'name = "vllm"' uv.lock | grep version | cut -d '"' -f 2)
+          VERSION=$(grep -A 1 "name = \"${{ matrix.backend }}\"" uv.lock | grep version | cut -d '"' -f 2)
           echo "version=$VERSION" >> $GITHUB_OUTPUT
 
       - name: Maximize build space
@@ -65,15 +70,16 @@ jobs:
         id: meta
         uses: docker/metadata-action@318604b99e75e41977312d83839a89be02ca4893
         with:
-          images: vectorinstitute/vector-inference
+          images: vectorinstitute/vector-inference-${{ matrix.backend }}
 
       - name: Build and push Docker image
         uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83
         with:
           context: .
-          file: ./Dockerfile
+          file: ./${{ matrix.backend }}.Dockerfile
           push: true
           tags: |
             ${{ steps.meta.outputs.tags }}
-            vectorinstitute/vector-inference:${{ steps.vllm-version.outputs.version }}
+            vectorinstitute/vector-inference-${{ matrix.backend }}:${{ steps.backend-version.outputs.version }}
+            vectorinstitute/vector-inference-${{ matrix.backend }}:latest
           labels: ${{ steps.meta.outputs.labels }}

pyproject.toml

@@ -36,19 +36,38 @@ docs = [
     "pymdown-extensions>=10.7.1",
     "mike>=2.0.0",
 ]
-
-[project.optional-dependencies]
-dev = [
+# Common inference dependencies (shared by both backends)
+inference = [
     "xgrammar>=0.1.11",
     "torch>=2.7.0",
-    "vllm>=0.10.0",
-    "ray[default]>=2.50.0",
     "cupy-cuda12x>=12.3.0",
     "flashinfer-python>=0.4.0",
-    "sglang>=0.5.0",
-    "orjson>=3.11.0",
+]
+
+[project.optional-dependencies]
+# vLLM inference backend (conflicts with sglang due to dependency version conflicts)
+# Install with: uv sync --extra vllm --group inference
+vllm = [
+    "vllm>=0.11.2",
+    "ray[default]>=2.51.0",
+]
+# SGLang inference backend (conflicts with vllm due to dependency version conflicts)
+# Install with: uv sync --extra sglang --group inference
+sglang = [
+    "sglang>=0.5.5",
     "sgl_kernel>=0.3.0",
-    "torchao>=0.14.0"
+    "orjson>=3.11.0",
+    "torchao>=0.9.0",
+]
+
+[tool.uv]
+# Declare that vllm and sglang extras are mutually exclusive
+# This prevents uv from trying to resolve both together in the lock file
+conflicts = [
+    [
+        { extra = "vllm" },
+        { extra = "sglang" },
+    ],
 ]
 
 [project.scripts]

sglang.Dockerfile

@@ -0,0 +1,70 @@
+FROM nvidia/cuda:12.8.1-cudnn-devel-ubuntu24.04
+
+# Non-interactive apt-get commands
+ARG DEBIAN_FRONTEND=noninteractive
+
+# No GPUs visible during build
+ARG CUDA_VISIBLE_DEVICES=none
+
+# Specify CUDA architectures -> 7.5: Quadro RTX 6000 & T4, 8.0: A100, 8.6: A40, 8.9: L40S, 9.0: H100
+ARG TORCH_CUDA_ARCH_LIST="7.5;8.0;8.6;8.9;9.0+PTX"
+
+# Set the Python version
+ARG PYTHON_VERSION=3.12.12
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    wget build-essential libssl-dev zlib1g-dev libbz2-dev \
+    libreadline-dev libsqlite3-dev libffi-dev libncursesw5-dev \
+    xz-utils tk-dev libxml2-dev libxmlsec1-dev liblzma-dev libnuma1 \
+    git vim \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Python
+RUN wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz && \
+    tar -xzf Python-$PYTHON_VERSION.tgz && \
+    cd Python-$PYTHON_VERSION && \
+    ./configure --enable-optimizations && \
+    make -j$(nproc) && \
+    make altinstall && \
+    cd .. && \
+    rm -rf Python-$PYTHON_VERSION.tgz Python-$PYTHON_VERSION
+
+# Install pip and core Python tools
+RUN wget https://bootstrap.pypa.io/get-pip.py && \
+    python3.12 get-pip.py && \
+    rm get-pip.py && \
+    python3.12 -m pip install --upgrade pip setuptools wheel uv
+
+# Install RDMA support
+RUN apt-get update && apt-get install -y \
+    libibverbs1 libibverbs-dev ibverbs-utils \
+    librdmacm1 librdmacm-dev rdmacm-utils \
+    rdma-core ibverbs-providers infiniband-diags perftest \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set up RDMA environment (these will persist in the final container)
+ENV LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH"
+ENV NCCL_IB_DISABLE=0
+ENV NCCL_SOCKET_IFNAME="^lo,docker0"
+ENV NCCL_NET_GDR_LEVEL=PHB
+ENV NCCL_IB_TIMEOUT=22
+ENV NCCL_IB_RETRY_CNT=7
+ENV NCCL_DEBUG=INFO
+
+# Set up project
+WORKDIR /vec-inf
+COPY . /vec-inf
+
+# Install project dependencies with sglang backend and inference group
+# Use --no-cache to prevent uv from storing both downloaded and extracted packages
+RUN uv pip install --system -e .[sglang] --group inference --prerelease=allow --no-cache && \
+    rm -rf /root/.cache/uv /tmp/*
+
+# Install a single, system NCCL (from NVIDIA CUDA repo in base image)
+RUN apt-get update && apt-get install -y --allow-change-held-packages\
+    libnccl2 libnccl-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set the default command to start an interactive shell
+CMD ["bash"]