Skip to content

Commit f5d2260

Browse files
XkunWXkunW
committed
Fix PP error, added metrics command
1 parent e7ed931 commit f5d2260

File tree

4 files changed

+127
-70
lines changed

4 files changed

+127
-70
lines changed

‎vec_inf/cli/_cli.py‎

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
import os
2+
import time
23
from typing import Optional
34

45
import click
56
from rich.columns import Columns
67
from rich.console import Console
8+
from rich.live import Live
79
from rich.panel import Panel
810

911
import vec_inf.cli._utils as utils
@@ -61,8 +63,8 @@ def cli():
6163
)
6264
@click.option(
6365
"--pipeline-parallelism",
64-
type=bool,
65-
help="Enable pipeline parallelism, default to True",
66+
type=str,
67+
help="Enable pipeline parallelism, accepts 'true' or 'false', defaults to 'true' for supported models"
6668
)
6769
@click.option(
6870
"--json-mode",
@@ -83,12 +85,15 @@ def launch(
8385
data_type: Optional[str] = None,
8486
venv: Optional[str] = None,
8587
log_dir: Optional[str] = None,
86-
pipeline_parallelism: Optional[bool] = True,
88+
pipeline_parallelism: Optional[str] = None,
8789
json_mode: bool = False,
8890
) -> None:
8991
"""
9092
Launch a model on the cluster
9193
"""
94+
95+
pipeline_parallelism = pipeline_parallelism is None or pipeline_parallelism.lower() == "true"
96+
9297
launch_script_path = os.path.join(
9398
os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "launch_server.sh"
9499
)
@@ -111,6 +116,8 @@ def launch(
111116
renamed_arg = arg.replace("_", "-")
112117
launch_cmd += f" --{renamed_arg} {locals()[arg]}"
113118

119+
print(launch_cmd)
120+
114121
output = utils.run_bash_command(launch_cmd)
115122

116123
slurm_job_id = output.split(" ")[-1].strip().strip("\n")
@@ -267,5 +274,32 @@ def list(model_name: Optional[str] = None, json_mode: bool = False) -> None:
267274
CONSOLE.print(Columns(panels, equal=True))
268275

269276

277+
@cli.command("metrics")
278+
@click.argument("slurm_job_id", type=int, nargs=1)
279+
@click.option(
280+
"--log-dir",
281+
type=str,
282+
help="Path to slurm log directory. This is required if --log-dir was set in model launch",
283+
)
284+
def metrics(slurm_job_id: int, log_dir: Optional[str] = None) -> None:
285+
"""
286+
Get metrics of a running model on the cluster
287+
"""
288+
status_cmd = f"scontrol show job {slurm_job_id} --oneliner"
289+
output = utils.run_bash_command(status_cmd)
290+
slurm_job_name = output.split(" ")[1].split("=")[1]
291+
out_logs = utils.read_slurm_log(slurm_job_name, slurm_job_id, "out", log_dir)
292+
293+
with Live(refresh_per_second=1, console=CONSOLE) as live:
294+
while True:
295+
metrics = utils.get_latest_metric(out_logs)
296+
table = utils.create_table(key_title="Metric", value_title="Value")
297+
for key, value in metrics.items():
298+
table.add_row(key, value)
299+
300+
live.update(table)
301+
302+
time.sleep(10)
303+
270304
if __name__ == "__main__":
271305
cli()

‎vec_inf/cli/_utils.py‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
import requests
77
from rich.table import Table
88

9-
MODEL_READY_SIGNATURE = "INFO: Uvicorn running on http://0.0.0.0:"
9+
MODEL_READY_SIGNATURE = "INFO: Application startup complete."
1010
SERVER_ADDRESS_SIGNATURE = "Server address: "
1111

1212

@@ -25,7 +25,7 @@ def read_slurm_log(
2525
slurm_job_name: str, slurm_job_id: int, slurm_log_type: str, log_dir: Optional[str]
2626
) -> Union[list[str], str]:
2727
"""
28-
Get the directory of a model
28+
Read the slurm log file
2929
"""
3030
if not log_dir:
3131
models_dir = os.path.join(os.path.expanduser("~"), ".vec-inf-logs")
@@ -135,3 +135,23 @@ def load_default_args(models_df: pd.DataFrame, model_name: str) -> dict:
135135
default_args = row_data.iloc[0].to_dict()
136136
default_args.pop("model_name")
137137
return default_args
138+
139+
140+
def get_latest_metric(log_lines: list[str]) -> dict | str:
141+
"""Read the latest metric entry from the log file."""
142+
latest_metric = {}
143+
144+
try:
145+
for line in reversed(log_lines):
146+
if "Avg prompt throughput" in line:
147+
# Parse the metric values from the line
148+
metrics = line.split("] ")[1].strip().strip(".")
149+
metrics = metrics.split(", ")
150+
for metric in metrics:
151+
key, value = metric.split(": ")
152+
latest_metric[key] = value
153+
break
154+
except Exception as e:
155+
return f"[red]Error reading log file: {e}[/red]"
156+
157+
return latest_metric

‎vec_inf/launch_server.sh‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,10 @@ export NUM_NODES=$num_nodes
4040
export NUM_GPUS=$num_gpus
4141
export VLLM_MAX_MODEL_LEN=$max_model_len
4242
export VLLM_MAX_LOGPROBS=$vocab_size
43-
export PIPLELINE_PARALLELISM=$pipeline_parallelism
43+
export PIPELINE_PARALLELISM=$pipeline_parallelism
44+
45+
echo Pipeline Parallelism: $PIPELINE_PARALLELISM
46+
4447
# For custom models, the following are set to default if not specified
4548
export VLLM_DATA_TYPE="auto"
4649
export VENV_BASE="singularity"

‎vec_inf/models/models.csv‎

Lines changed: 64 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1,65 +1,65 @@
11
model_name,model_family,model_variant,partition,qos,time,num_gpus,num_nodes,vocab_size,max_model_len,data_type,venv,log_dir,pipeline_parallelism
2-
c4ai-command-r-plus,c4ai-command-r,plus,a40,m2,08:00:00,4,2,256000,8192,auto,singularity,default,True
3-
c4ai-command-r-plus-08-2024,c4ai-command-r,plus-08-2024,a40,m2,08:00:00,4,2,256000,65536,auto,singularity,default,True
4-
c4ai-command-r-08-2024,c4ai-command-r,08-2024,a40,m2,08:00:00,2,1,256000,32768,auto,singularity,default,True
5-
CodeLlama-7b-hf,CodeLlama,7b-hf,a40,m2,08:00:00,1,1,32000,16384,auto,singularity,default,True
6-
CodeLlama-7b-Instruct-hf,CodeLlama,7b-Instruct-hf,a40,m2,08:00:00,1,1,32000,16384,auto,singularity,default,True
7-
CodeLlama-13b-hf,CodeLlama,13b-hf,a40,m2,08:00:00,1,1,32000,16384,auto,singularity,default,True
8-
CodeLlama-13b-Instruct-hf,CodeLlama,13b-Instruct-hf,a40,m2,08:00:00,1,1,32000,16384,auto,singularity,default,True
9-
CodeLlama-34b-hf,CodeLlama,34b-hf,a40,m2,08:00:00,2,1,32000,16384,auto,singularity,default,True
10-
CodeLlama-34b-Instruct-hf,CodeLlama,34b-Instruct-hf,a40,m2,08:00:00,2,1,32000,16384,auto,singularity,default,True
11-
CodeLlama-70b-hf,CodeLlama,70b-hf,a40,m2,08:00:00,4,1,32000,4096,auto,singularity,default,True
12-
CodeLlama-70b-Instruct-hf,CodeLlama,70b-Instruct-hf,a40,m2,08:00:00,4,1,32000,4096,auto,singularity,default,True
13-
dbrx-instruct,dbrx,instruct,a40,m2,08:00:00,4,2,100352,32000,auto,singularity,default,True
14-
gemma-2-9b,gemma-2,9b,a40,m2,08:00:00,1,1,256000,4096,auto,singularity,default,True
15-
gemma-2-9b-it,gemma-2,9b-it,a40,m2,08:00:00,1,1,256000,4096,auto,singularity,default,True
16-
gemma-2-27b,gemma-2,27b,a40,m2,08:00:00,2,1,256000,4096,auto,singularity,default,True
17-
gemma-2-27b-it,gemma-2,27b-it,a40,m2,08:00:00,2,1,256000,4096,auto,singularity,default,True
18-
Llama-2-7b-hf,Llama-2,7b-hf,a40,m2,08:00:00,1,1,32000,4096,auto,singularity,default,True
19-
Llama-2-7b-chat-hf,Llama-2,7b-chat-hf,a40,m2,08:00:00,1,1,32000,4096,auto,singularity,default,True
20-
Llama-2-13b-hf,Llama-2,13b-hf,a40,m2,08:00:00,1,1,32000,4096,auto,singularity,default,True
21-
Llama-2-13b-chat-hf,Llama-2,13b-chat-hf,a40,m2,08:00:00,1,1,32000,4096,auto,singularity,default,True
22-
Llama-2-70b-hf,Llama-2,70b-hf,a40,m2,08:00:00,4,1,32000,4096,auto,singularity,default,True
23-
Llama-2-70b-chat-hf,Llama-2,70b-chat-hf,a40,m2,08:00:00,4,1,32000,4096,auto,singularity,default,True
24-
llava-1.5-7b-hf,llava-1.5,7b-hf,a40,m2,08:00:00,1,1,32000,4096,auto,singularity,default,True
25-
llava-1.5-13b-hf,llava-1.5,13b-hf,a40,m2,08:00:00,1,1,32000,4096,auto,singularity,default,True
26-
llava-v1.6-mistral-7b-hf,llava-v1.6,mistral-7b-hf,a40,m2,08:00:00,1,1,32064,32768,auto,singularity,default,True
27-
llava-v1.6-34b-hf,llava-v1.6,34b-hf,a40,m2,08:00:00,2,1,64064,4096,auto,singularity,default,True
28-
Meta-Llama-3-8B,Meta-Llama-3,8B,a40,m2,08:00:00,1,1,128256,8192,auto,singularity,default,True
29-
Meta-Llama-3-8B-Instruct,Meta-Llama-3,8B-Instruct,a40,m2,08:00:00,1,1,128256,8192,auto,singularity,default,True
30-
Meta-Llama-3-70B,Meta-Llama-3,70B,a40,m2,08:00:00,4,1,128256,8192,auto,singularity,default,True
31-
Meta-Llama-3-70B-Instruct,Meta-Llama-3,70B-Instruct,a40,m2,08:00:00,4,1,128256,8192,auto,singularity,default,True
32-
Meta-Llama-3.1-8B,Meta-Llama-3.1,8B,a40,m2,08:00:00,1,1,128256,131072,auto,singularity,default,True
33-
Meta-Llama-3.1-8B-Instruct,Meta-Llama-3.1,8B-Instruct,a40,m2,08:00:00,1,1,128256,131072,auto,singularity,default,True
34-
Meta-Llama-3.1-70B,Meta-Llama-3.1,70B,a40,m2,08:00:00,4,1,128256,65536,auto,singularity,default,True
35-
Meta-Llama-3.1-70B-Instruct,Meta-Llama-3.1,70B-Instruct,a40,m2,08:00:00,4,1,128256,65536,auto,singularity,default,True
36-
Meta-Llama-3.1-405B-Instruct,Meta-Llama-3.1,405B-Instruct,a40,m4,02:00:00,4,8,128256,16384,auto,singularity,default,True
37-
Mistral-7B-v0.1,Mistral,7B-v0.1,a40,m2,08:00:00,1,1,32000,32768,auto,singularity,default,True
38-
Mistral-7B-Instruct-v0.1,Mistral,7B-Instruct-v0.1,a40,m2,08:00:00,1,1,32000,32768,auto,singularity,default,True
39-
Mistral-7B-Instruct-v0.2,Mistral,7B-Instruct-v0.2,a40,m2,08:00:00,1,1,32000,32768,auto,singularity,default,True
40-
Mistral-7B-v0.3,Mistral,7B-v0.3,a40,m2,08:00:00,1,1,32768,32768,auto,singularity,default,True
41-
Mistral-7B-Instruct-v0.3,Mistral,7B-Instruct-v0.3,a40,m2,08:00:00,1,1,32768,32768,auto,singularity,default,True
42-
Mistral-Large-Instruct-2407,Mistral,Large-Instruct-2407,a40,m2,08:00:00,4,2,32768,32768,auto,singularity,default,True
43-
Mixtral-8x7B-Instruct-v0.1,Mixtral,8x7B-Instruct-v0.1,a40,m2,08:00:00,4,1,32000,32768,auto,singularity,default,True
44-
Mixtral-8x22B-v0.1,Mixtral,8x22B-v0.1,a40,m2,08:00:00,4,2,32768,65536,auto,singularity,default,True
45-
Mixtral-8x22B-Instruct-v0.1,Mixtral,8x22B-Instruct-v0.1,a40,m2,08:00:00,4,2,32768,65536,auto,singularity,default,True
46-
Phi-3-medium-128k-instruct,Phi-3,medium-128k-instruct,a40,m2,08:00:00,2,1,32064,131072,auto,singularity,default,True
47-
Phi-3-vision-128k-instruct,Phi-3,vision-128k-instruct,a40,m2,08:00:00,2,1,32064,65536,auto,singularity,default,True
48-
Llama3-OpenBioLLM-70B,Llama3-OpenBioLLM,70B,a40,m2,08:00:00,4,1,128256,8192,auto,singularity,default,True
49-
Llama-3.1-Nemotron-70B-Instruct-HF,Llama-3.1-Nemotron,70B-Instruct-HF,a40,m2,08:00:00,4,1,128256,65536,auto,singularity,default,True
50-
Llama-3.2-1B,Llama-3.2,1B,a40,m2,08:00:00,1,1,128256,131072,auto,singularity,default,True
51-
Llama-3.2-1B-Instruct,Llama-3.2,1B-Instruct,a40,m2,08:00:00,1,1,128256,131072,auto,singularity,default,True
52-
Llama-3.2-3B,Llama-3.2,3B,a40,m2,08:00:00,1,1,128256,131072,auto,singularity,default,True
53-
Llama-3.2-3B-Instruct,Llama-3.2,3B-Instruct,a40,m2,08:00:00,1,1,128256,131072,auto,singularity,default,True
54-
Llama-3.2-11B-Vision,Llama-3.2,11B-Vision,a40,m2,08:00:00,1,1,128256,32768,auto,singularity,default,True
55-
Llama-3.2-11B-Vision-Instruct,Llama-3.2,11B-Vision-Instruct,a40,m2,08:00:00,1,1,128256,32768,auto,singularity,default,True
56-
Llama-3.2-90B-Vision,Llama-3.2,90B-Vision,a40,m2,08:00:00,4,2,128256,8192,auto,singularity,default,True
57-
Llama-3.2-90B-Vision-Instruct,Llama-3.2,90B-Vision-Instruct,a40,m2,08:00:00,4,2,128256,8192,auto,singularity,default,True
58-
Qwen2.5-0.5B-Instruct,Qwen2.5,0.5B-Instruct,a40,m2,08:00:00,1,1,152064,32768,auto,singularity,default,True
59-
Qwen2.5-1.5B-Instruct,Qwen2.5,1.5B-Instruct,a40,m2,08:00:00,1,1,152064,32768,auto,singularity,default,True
60-
Qwen2.5-3B-Instruct,Qwen2.5,3B-Instruct,a40,m2,08:00:00,1,1,152064,32768,auto,singularity,default,True
61-
Qwen2.5-7B-Instruct,Qwen2.5,7B-Instruct,a40,m2,08:00:00,1,1,152064,32768,auto,singularity,default,True
62-
Qwen2.5-14B-Instruct,Qwen2.5,14B-Instruct,a40,m2,08:00:00,1,1,152064,32768,auto,singularity,default,True
63-
Qwen2.5-32B-Instruct,Qwen2.5,32B-Instruct,a40,m2,08:00:00,2,1,152064,32768,auto,singularity,default,True
64-
Qwen2.5-72B-Instruct,Qwen2.5,72B-Instruct,a40,m2,08:00:00,4,1,152064,32768,auto,singularity,default,True
65-
Pixtral-12B-2409,Pixtral,12B-2409,a40,m2,08:00:00,1,1,131072,16384,auto,singularity,default,True
2+
c4ai-command-r-plus,c4ai-command-r,plus,a40,m2,08:00:00,4,2,256000,8192,auto,singularity,default,true
3+
c4ai-command-r-plus-08-2024,c4ai-command-r,plus-08-2024,a40,m2,08:00:00,4,2,256000,65536,auto,singularity,default,true
4+
c4ai-command-r-08-2024,c4ai-command-r,08-2024,a40,m2,08:00:00,2,1,256000,32768,auto,singularity,default,true
5+
CodeLlama-7b-hf,CodeLlama,7b-hf,a40,m2,08:00:00,1,1,32000,16384,auto,singularity,default,true
6+
CodeLlama-7b-Instruct-hf,CodeLlama,7b-Instruct-hf,a40,m2,08:00:00,1,1,32000,16384,auto,singularity,default,true
7+
CodeLlama-13b-hf,CodeLlama,13b-hf,a40,m2,08:00:00,1,1,32000,16384,auto,singularity,default,true
8+
CodeLlama-13b-Instruct-hf,CodeLlama,13b-Instruct-hf,a40,m2,08:00:00,1,1,32000,16384,auto,singularity,default,true
9+
CodeLlama-34b-hf,CodeLlama,34b-hf,a40,m2,08:00:00,2,1,32000,16384,auto,singularity,default,true
10+
CodeLlama-34b-Instruct-hf,CodeLlama,34b-Instruct-hf,a40,m2,08:00:00,2,1,32000,16384,auto,singularity,default,true
11+
CodeLlama-70b-hf,CodeLlama,70b-hf,a40,m2,08:00:00,4,1,32000,4096,auto,singularity,default,true
12+
CodeLlama-70b-Instruct-hf,CodeLlama,70b-Instruct-hf,a40,m2,08:00:00,4,1,32000,4096,auto,singularity,default,true
13+
dbrx-instruct,dbrx,instruct,a40,m2,08:00:00,4,2,100352,32000,auto,singularity,default,true
14+
gemma-2-9b,gemma-2,9b,a40,m2,08:00:00,1,1,256000,4096,auto,singularity,default,true
15+
gemma-2-9b-it,gemma-2,9b-it,a40,m2,08:00:00,1,1,256000,4096,auto,singularity,default,true
16+
gemma-2-27b,gemma-2,27b,a40,m2,08:00:00,2,1,256000,4096,auto,singularity,default,true
17+
gemma-2-27b-it,gemma-2,27b-it,a40,m2,08:00:00,2,1,256000,4096,auto,singularity,default,true
18+
Llama-2-7b-hf,Llama-2,7b-hf,a40,m2,08:00:00,1,1,32000,4096,auto,singularity,default,true
19+
Llama-2-7b-chat-hf,Llama-2,7b-chat-hf,a40,m2,08:00:00,1,1,32000,4096,auto,singularity,default,true
20+
Llama-2-13b-hf,Llama-2,13b-hf,a40,m2,08:00:00,1,1,32000,4096,auto,singularity,default,true
21+
Llama-2-13b-chat-hf,Llama-2,13b-chat-hf,a40,m2,08:00:00,1,1,32000,4096,auto,singularity,default,true
22+
Llama-2-70b-hf,Llama-2,70b-hf,a40,m2,08:00:00,4,1,32000,4096,auto,singularity,default,true
23+
Llama-2-70b-chat-hf,Llama-2,70b-chat-hf,a40,m2,08:00:00,4,1,32000,4096,auto,singularity,default,true
24+
llava-1.5-7b-hf,llava-1.5,7b-hf,a40,m2,08:00:00,1,1,32000,4096,auto,singularity,default,true
25+
llava-1.5-13b-hf,llava-1.5,13b-hf,a40,m2,08:00:00,1,1,32000,4096,auto,singularity,default,true
26+
llava-v1.6-mistral-7b-hf,llava-v1.6,mistral-7b-hf,a40,m2,08:00:00,1,1,32064,32768,auto,singularity,default,true
27+
llava-v1.6-34b-hf,llava-v1.6,34b-hf,a40,m2,08:00:00,2,1,64064,4096,auto,singularity,default,true
28+
Meta-Llama-3-8B,Meta-Llama-3,8B,a40,m2,08:00:00,1,1,128256,8192,auto,singularity,default,true
29+
Meta-Llama-3-8B-Instruct,Meta-Llama-3,8B-Instruct,a40,m2,08:00:00,1,1,128256,8192,auto,singularity,default,true
30+
Meta-Llama-3-70B,Meta-Llama-3,70B,a40,m2,08:00:00,4,1,128256,8192,auto,singularity,default,true
31+
Meta-Llama-3-70B-Instruct,Meta-Llama-3,70B-Instruct,a40,m2,08:00:00,4,1,128256,8192,auto,singularity,default,true
32+
Meta-Llama-3.1-8B,Meta-Llama-3.1,8B,a40,m2,08:00:00,1,1,128256,131072,auto,singularity,default,true
33+
Meta-Llama-3.1-8B-Instruct,Meta-Llama-3.1,8B-Instruct,a40,m2,08:00:00,1,1,128256,131072,auto,singularity,default,true
34+
Meta-Llama-3.1-70B,Meta-Llama-3.1,70B,a40,m2,08:00:00,4,1,128256,65536,auto,singularity,default,true
35+
Meta-Llama-3.1-70B-Instruct,Meta-Llama-3.1,70B-Instruct,a40,m2,08:00:00,4,1,128256,65536,auto,singularity,default,true
36+
Meta-Llama-3.1-405B-Instruct,Meta-Llama-3.1,405B-Instruct,a40,m4,02:00:00,4,8,128256,16384,auto,singularity,default,true
37+
Mistral-7B-v0.1,Mistral,7B-v0.1,a40,m2,08:00:00,1,1,32000,32768,auto,singularity,default,true
38+
Mistral-7B-Instruct-v0.1,Mistral,7B-Instruct-v0.1,a40,m2,08:00:00,1,1,32000,32768,auto,singularity,default,true
39+
Mistral-7B-Instruct-v0.2,Mistral,7B-Instruct-v0.2,a40,m2,08:00:00,1,1,32000,32768,auto,singularity,default,true
40+
Mistral-7B-v0.3,Mistral,7B-v0.3,a40,m2,08:00:00,1,1,32768,32768,auto,singularity,default,true
41+
Mistral-7B-Instruct-v0.3,Mistral,7B-Instruct-v0.3,a40,m2,08:00:00,1,1,32768,32768,auto,singularity,default,true
42+
Mistral-Large-Instruct-2407,Mistral,Large-Instruct-2407,a40,m2,08:00:00,4,2,32768,32768,auto,singularity,default,true
43+
Mixtral-8x7B-Instruct-v0.1,Mixtral,8x7B-Instruct-v0.1,a40,m2,08:00:00,4,1,32000,32768,auto,singularity,default,true
44+
Mixtral-8x22B-v0.1,Mixtral,8x22B-v0.1,a40,m2,08:00:00,4,2,32768,65536,auto,singularity,default,true
45+
Mixtral-8x22B-Instruct-v0.1,Mixtral,8x22B-Instruct-v0.1,a40,m2,08:00:00,4,2,32768,65536,auto,singularity,default,true
46+
Phi-3-medium-128k-instruct,Phi-3,medium-128k-instruct,a40,m2,08:00:00,2,1,32064,131072,auto,singularity,default,true
47+
Phi-3-vision-128k-instruct,Phi-3,vision-128k-instruct,a40,m2,08:00:00,2,1,32064,65536,auto,singularity,default,true
48+
Llama3-OpenBioLLM-70B,Llama3-OpenBioLLM,70B,a40,m2,08:00:00,4,1,128256,8192,auto,singularity,default,true
49+
Llama-3.1-Nemotron-70B-Instruct-HF,Llama-3.1-Nemotron,70B-Instruct-HF,a40,m2,08:00:00,4,1,128256,65536,auto,singularity,default,true
50+
Llama-3.2-1B,Llama-3.2,1B,a40,m2,08:00:00,1,1,128256,131072,auto,singularity,default,true
51+
Llama-3.2-1B-Instruct,Llama-3.2,1B-Instruct,a40,m2,08:00:00,1,1,128256,131072,auto,singularity,default,true
52+
Llama-3.2-3B,Llama-3.2,3B,a40,m2,08:00:00,1,1,128256,131072,auto,singularity,default,true
53+
Llama-3.2-3B-Instruct,Llama-3.2,3B-Instruct,a40,m2,08:00:00,1,1,128256,131072,auto,singularity,default,true
54+
Llama-3.2-11B-Vision,Llama-3.2,11B-Vision,a40,m2,08:00:00,1,1,128256,32768,auto,singularity,default,true
55+
Llama-3.2-11B-Vision-Instruct,Llama-3.2,11B-Vision-Instruct,a40,m2,08:00:00,1,1,128256,32768,auto,singularity,default,true
56+
Llama-3.2-90B-Vision,Llama-3.2,90B-Vision,a40,m2,08:00:00,4,2,128256,8192,auto,singularity,default,true
57+
Llama-3.2-90B-Vision-Instruct,Llama-3.2,90B-Vision-Instruct,a40,m2,08:00:00,4,2,128256,8192,auto,singularity,default,true
58+
Qwen2.5-0.5B-Instruct,Qwen2.5,0.5B-Instruct,a40,m2,08:00:00,1,1,152064,32768,auto,singularity,default,true
59+
Qwen2.5-1.5B-Instruct,Qwen2.5,1.5B-Instruct,a40,m2,08:00:00,1,1,152064,32768,auto,singularity,default,true
60+
Qwen2.5-3B-Instruct,Qwen2.5,3B-Instruct,a40,m2,08:00:00,1,1,152064,32768,auto,singularity,default,true
61+
Qwen2.5-7B-Instruct,Qwen2.5,7B-Instruct,a40,m2,08:00:00,1,1,152064,32768,auto,singularity,default,true
62+
Qwen2.5-14B-Instruct,Qwen2.5,14B-Instruct,a40,m2,08:00:00,1,1,152064,32768,auto,singularity,default,true
63+
Qwen2.5-32B-Instruct,Qwen2.5,32B-Instruct,a40,m2,08:00:00,2,1,152064,32768,auto,singularity,default,true
64+
Qwen2.5-72B-Instruct,Qwen2.5,72B-Instruct,a40,m2,08:00:00,4,1,152064,32768,auto,singularity,default,true
65+
Pixtral-12B-2409,Pixtral,12B-2409,a40,m2,08:00:00,1,1,131072,16384,auto,singularity,default,true

0 commit comments

Comments
 (0)