Spaces:

moonshotai
/

Kimi-Dev-72B

Running on L40S

miaoyibo commited on 8 days ago

Commit

3425837

1 Parent(s): eca802f

1

Files changed (1) hide show

serve_vllm.sh CHANGED Viewed

@@ -3,8 +3,8 @@
 python -m vllm.entrypoints.openai.api_server \
     --model moonshotai/Kimi-Dev-72B \
     --tensor-parallel-size 4 \
-    --max-num-seqs 8 \
-    --max-model-len 131072 \
     --gpu-memory-utilization 0.9 \
     --host localhost \
     --served-model-name kimi-dev \

 python -m vllm.entrypoints.openai.api_server \
     --model moonshotai/Kimi-Dev-72B \
     --tensor-parallel-size 4 \
+    --max-num-seqs 4 \
+    --max-model-len 65536 \
     --gpu-memory-utilization 0.9 \
     --host localhost \
     --served-model-name kimi-dev \