a

kimi_dev/serve/inference.py

@@ -5,7 +5,6 @@ from transformers import (
     AutoConfig,
     AutoTokenizer
 )
-import tensor_parallel as tp
 
 logger = logging.getLogger(__name__)
 
@@ -21,8 +20,6 @@ def load_model(model_path: str = "moonshotai/Kimi-Dev-72B"):
         device_map="auto",
         trust_remote_code=True,
     )
-    model = tp.tensor_parallel(model, ["cuda:0", "cuda:1", "cuda:2", "cuda:3"])  # <- each GPU has half the weights
-
 
     tokenizer = AutoTokenizer.from_pretrained(model_path)
 

requirements.txt

@@ -17,5 +17,4 @@ tqdm
 colorama
 Pygments
 markdown
-SentencePiece
-tensor_parallel
+SentencePiece