meituan

/

DeepSeek-R1-Block-INT8

like 44

Follow

meituan 131

• Startup command :

python3 -m sglang.launch_server \
    --model /model/quantized_model/DeepSeek-R1-block-int8 \
    --trust-remote-code --mem-fraction-static 0.95 --max-running-requests 1 \
    --served-model-name deepseek --port 4396 --context-length 1024 \
    --disable-radix --tp 8

• First curl command :

curl --location 'http://127.0.0.1:4396/v1/chat/completions' --header 'Content-Type: application/json' --data '{
    "max_tokens": 1000, 
    "messages" : [{"role": "user", "content": "How to quantize a moe model with 671B params？"}], 
    "model": "deepseek", 
    "temperature": 0.7, 
    "top_k": 30,
    "top_p": 0.3, 
    "stream": false
}'

• response :

{"id":"c93bbe91bb2f4fc393367f451a3fc705","object":"chat.completion","created":1740986677,"model":"deepseek","choices":[{"index":0,"message":{"role":"assistant","content":"ſſ{{{ſ{{{ſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſſ","tool_calls":null},"logprobs":null,"finish_reason":"length","matched_stop":null}],"usage":{"prompt_tokens":17,"total_tokens":1017,"completion_tokens":1000,"prompt_tokens_details":null}}

• Second curl command :

curl --location 'http://127.0.0.1:4396/v1/completions' \
--header 'Content-Type: application/json' \
--data '{
    "model": "deepseek",
    "prompt": "How to quantize a moe model with 671B params？",
    "max_tokens": 512,
    "temperature": 0.7,
    "stream": 0
  }'

• response :

{"id":"d7537edd2ccb468ea5cc56f2095c02df","object":"text_completion","created":1740987263,"model":"deepseek","choices":[{"index":0,"text":"­ ‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑‑","logprobs":null,"finish_reason":"length","matched_stop":null}],"usage":{"prompt_tokens":15,"total_tokens":527,"completion_tokens":512,"prompt_tokens_details":null}}

In addition to the above situation, sometimes the response will also consist entirely of empty characters " " to the max_tokens.

@pkumc @HandH1998 @yuanzu plz check this issue.

I met the same problem when I directly run the "bf16_cast_block_int8.py" from fp8 tensors (I commented out the "assert ....") . I fixed this by first turn the fp8 tensors to fp16 (code in deepseek v3 git repo), then turn it to int8.

Yes, it need to convert to bf16 from fp8 using the deepseek official code. @yzhou992 @ShiningMaker