Add suport for enforce eager, remove URL file creation

Author: XkunW

vec_inf/launch_server.sh

@@ -19,6 +19,7 @@ while [[ "$#" -gt 0 ]]; do
         --log-dir) log_dir="$2"; shift ;;
         --model-weights-parent-dir) model_weights_parent_dir="$2"; shift ;;
         --pipeline-parallelism) pipeline_parallelism="$2"; shift ;;
+        --enforce-eager) enforce_eager="$2"; shift ;;
         *) echo "Unknown parameter passed: $1"; exit 1 ;;
     esac
     shift
@@ -59,6 +60,12 @@ else
     export PIPELINE_PARALLELISM="False"
 fi
 
+if [ -n "$enforce_eager" ]; then
+    export ENFORCE_EAGER=$enforce_eager
+else
+    export ENFORCE_EAGER="False"
+fi
+
 # ================================= Set default environment variables ======================================
 # Slurm job configuration
 export JOB_NAME="$MODEL_FAMILY-$MODEL_VARIANT"
@@ -68,10 +75,9 @@ fi
 mkdir -p $LOG_DIR
 
 # Model and entrypoint configuration. API Server URL (host, port) are set automatically based on the
-# SLURM job and are written to the file specified at VLLM_BASE_URL_FILENAME
+# SLURM job 
 export SRC_DIR="$(dirname "$0")"
 export MODEL_DIR="${SRC_DIR}/models/${MODEL_FAMILY}"
-export VLLM_BASE_URL_FILENAME="${MODEL_DIR}/.${JOB_NAME}_url"
 
 # Variables specific to your working environment, below are examples for the Vector cluster
 export VLLM_MODEL_WEIGHTS="${MODEL_WEIGHTS_PARENT_DIR}/${JOB_NAME}"
@@ -89,11 +95,6 @@ if [[ $fp16_partitions =~ $JOB_PARTITION ]]; then
     echo "Data type set to due to non-Ampere GPUs used: $VLLM_DATA_TYPE"
 fi
 
-# Create a file to store the API server URL if it doesn't exist
-if [ -f $VLLM_BASE_URL_FILENAME ]; then
-    touch $VLLM_BASE_URL_FILENAME
-fi
-
 echo Job Name: $JOB_NAME
 echo Partition: $JOB_PARTITION
 echo Num Nodes: $NUM_NODES
@@ -105,6 +106,7 @@ echo Max Model Length: $VLLM_MAX_MODEL_LEN
 echo Max Num Seqs: $VLLM_MAX_NUM_SEQS
 echo Vocabulary Size: $VLLM_MAX_LOGPROBS
 echo Pipeline Parallelism: $PIPELINE_PARALLELISM
+echo Enforce Eager: $ENFORCE_EAGER
 echo Log Directory: $LOG_DIR
 echo Model Weights Parent Directory: $MODEL_WEIGHTS_PARENT_DIR
 

vec_inf/multinode_vllm.slurm

@@ -12,7 +12,7 @@ nvidia-smi
 source ${SRC_DIR}/find_port.sh
 
 if [ "$VENV_BASE" = "singularity" ]; then
-    export SINGULARITY_IMAGE=/projects/aieng/public/vector-inference_0.3.4.sif
+    export SINGULARITY_IMAGE=/projects/aieng/public/vector-inference_0.6.4.post1.sif
     export VLLM_NCCL_SO_PATH=/vec-inf/nccl/libnccl.so.2.18.1
     module load singularity-ce/3.8.2
     singularity exec $SINGULARITY_IMAGE ray stop
@@ -73,7 +73,6 @@ done
 vllm_port_number=$(find_available_port $head_node_ip 8080 65535)
 
 echo "Server address: http://${head_node_ip}:${vllm_port_number}/v1"
-echo "http://${head_node_ip}:${vllm_port_number}/v1" > ${VLLM_BASE_URL_FILENAME}
 
 if [ "$PIPELINE_PARALLELISM" = "True" ]; then
     export PIPELINE_PARALLEL_SIZE=$NUM_NODES
@@ -83,6 +82,12 @@ else
     export TENSOR_PARALLEL_SIZE=$((NUM_NODES*NUM_GPUS))
 fi
 
+if [ "$ENFORCE_EAGER" = "True" ]; then
+    export ENFORCE_EAGER="--enforce-eager"
+else
+    export ENFORCE_EAGER=""
+fi
+
 # Activate vllm venv
 if [ "$VENV_BASE" = "singularity" ]; then
     singularity exec --nv --bind ${MODEL_WEIGHTS_PARENT_DIR}:${MODEL_WEIGHTS_PARENT_DIR} $SINGULARITY_IMAGE \
@@ -97,7 +102,8 @@ if [ "$VENV_BASE" = "singularity" ]; then
     --trust-remote-code \
     --max-logprobs ${VLLM_MAX_LOGPROBS} \
     --max-model-len ${VLLM_MAX_MODEL_LEN} \
-    --max-num-seqs ${VLLM_MAX_NUM_SEQS}
+    --max-num-seqs ${VLLM_MAX_NUM_SEQS} \
+    ${ENFORCE_EAGER}
 else
     source ${VENV_BASE}/bin/activate
     python3 -m vllm.entrypoints.openai.api_server \
@@ -111,5 +117,6 @@ else
     --trust-remote-code \
     --max-logprobs ${VLLM_MAX_LOGPROBS} \
     --max-model-len ${VLLM_MAX_MODEL_LEN} \
-    --max-num-seqs ${VLLM_MAX_NUM_SEQS}
+    --max-num-seqs ${VLLM_MAX_NUM_SEQS} \
+    ${ENFORCE_EAGER}
 fi

vec_inf/vllm.slurm

@@ -14,11 +14,16 @@ hostname=${SLURMD_NODENAME}
 vllm_port_number=$(find_available_port $hostname 8080 65535)
 
 echo "Server address: http://${hostname}:${vllm_port_number}/v1"
-echo "http://${hostname}:${vllm_port_number}/v1" > ${VLLM_BASE_URL_FILENAME}
+
+if [ "$ENFORCE_EAGER" = "True" ]; then
+    export ENFORCE_EAGER="--enforce-eager"
+else
+    export ENFORCE_EAGER=""
+fi
 
 # Activate vllm venv
 if [ "$VENV_BASE" = "singularity" ]; then
-    export SINGULARITY_IMAGE=/projects/aieng/public/vector-inference_0.3.4.sif
+    export SINGULARITY_IMAGE=/projects/aieng/public/vector-inference_0.6.4.post1.sif
     export VLLM_NCCL_SO_PATH=/vec-inf/nccl/libnccl.so.2.18.1
     module load singularity-ce/3.8.2
     singularity exec $SINGULARITY_IMAGE ray stop
@@ -33,7 +38,8 @@ if [ "$VENV_BASE" = "singularity" ]; then
     --max-logprobs ${VLLM_MAX_LOGPROBS} \
     --trust-remote-code \
     --max-model-len ${VLLM_MAX_MODEL_LEN} \
-    --max-num-seqs ${VLLM_MAX_NUM_SEQS}
+    --max-num-seqs ${VLLM_MAX_NUM_SEQS} \
+    ${ENFORCE_EAGER}
 else
     source ${VENV_BASE}/bin/activate
     python3 -m vllm.entrypoints.openai.api_server \
@@ -46,5 +52,6 @@ else
     --max-logprobs ${VLLM_MAX_LOGPROBS} \
     --trust-remote-code \
     --max-model-len ${VLLM_MAX_MODEL_LEN} \
-    --max-num-seqs ${VLLM_MAX_NUM_SEQS}
+    --max-num-seqs ${VLLM_MAX_NUM_SEQS} \
+    ${ENFORCE_EAGER}
 fi