1

.gradio/certificate.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----

app.py

@@ -1,6 +1,7 @@
 import argparse
 import gradio as gr
 import os
+os.environ["HF_HOME"] = "/mnt/moonfs/miaoyibo-ksyun/hf_home"
 from PIL import Image
 import spaces
 import copy
@@ -27,7 +28,7 @@ from kimi_vl.serve.chat_utils import (
     to_gradio_chatbot,
     to_gradio_history,
 )
-from kimi_vl.serve.inference import kimi_vl_generate, load_model
+from kimi_vl.serve.inference import kimi_dev_generate, load_model
 from kimi_vl.serve.examples import get_examples
 
 TITLE = """<h1 align="left" style="min-width:200px; margin-top:0;">Chat with Kimi-Dev-72B🤔 </h1>"""
@@ -127,7 +128,7 @@ def predict(
     """
     print("running the prediction function")
     try:
-        model = fetch_model(args.model)
+        model, tokenizer = fetch_model(args.model)
 
         if text == "":
             yield chatbot, history, "Empty context."
@@ -136,6 +137,8 @@ def predict(
         yield [[text, "No Model Found"]], [], "No Model Found"
         return
 
+    
+
     if images is None:
         images = []
 
@@ -165,10 +168,11 @@ def predict(
     gradio_chatbot_output = to_gradio_chatbot(conversation)
 
     full_response = ""
-    for x in kimi_vl_generate(
+    for x in kimi_dev_generate(
             conversations=all_conv,
             model=model,
-            processor=processor,
+            tokneizer=tokenizer,
+            # processor=processor,
             stop_words=stop_words,
             max_length=max_length_tokens,
             temperature=temperature,
@@ -340,13 +344,18 @@ def main(args: argparse.Namespace):
 
     # concurrency_count=CONCURRENT_COUNT, max_size=MAX_EVENTS
     favicon_path = os.path.join("kimi_vl/serve/assets/favicon.ico")
+    # demo.queue().launch(
+    #     favicon_path=favicon_path,
+    #     server_name=args.ip,
+    #     server_port=args.port,
+    # )
     demo.queue().launch(
         favicon_path=favicon_path,
         server_name=args.ip,
         server_port=args.port,
+        share=True # for 本地调试
     )
 
-
 if __name__ == "__main__":
     args = parse_args()
     print(args)

kimi_vl/serve/inference.py

@@ -12,6 +12,7 @@ from transformers import (
     StoppingCriteria,
     StoppingCriteriaList,
     TextIteratorStreamer,
+    AutoTokenizer
 )
 
 from .chat_utils import Conversation, get_conv_template
@@ -35,8 +36,9 @@ def load_model(model_path: str = "moonshotai/Kimi-Dev-72B"):
         trust_remote_code=True,
     )
     # processor = AutoProcessor.from_pretrained(model_path, config=config, trust_remote_code=True)
+    tokenizer = AutoTokenizer.from_pretrained(model_path)
 
-    return model
+    return model, tokenizer
 
 
 class StoppingCriteriaSub(StoppingCriteria):
@@ -155,9 +157,10 @@ def preprocess(
 
 @torch.no_grad()
 @torch.inference_mode()
-def kimi_vl_generate(
+def kimi_dev_generate(
     model: torch.nn.Module,
-    processor: AutoProcessor,
+    tokenizer,
+    # processor: AutoProcessor,
     conversations: list[Conversation],
     stop_words: list,
     max_length: int = 256,
@@ -167,12 +170,12 @@ def kimi_vl_generate(
 ):
     # convert conversation to inputs
     print(f"conversations = {conversations}")
-    inputs = preprocess(conversations, processor=processor)
+    # inputs = preprocess(conversations)
+    inputs = tokenizer.tokenize(conversations)
     inputs = inputs.to(model.device)
 
     return generate(
         model,
-        processor,
         inputs,
         max_gen_len=max_length,
         temperature=temperature,