Update app.py

app.py

@@ -45,50 +45,61 @@ token=os.getenv('token')
 print('token = ',token)
 
 from transformers import AutoModelForCausalLM, AutoTokenizer
+import transformers
 
 # model_id = "mistralai/Mistral-7B-v0.3"
 
-model_id = "microsoft/Phi-3-medium-4k-instruct"
-model_id = "microsoft/phi-4"
+# model_id = "microsoft/Phi-3-medium-4k-instruct"
+# model_id = "microsoft/phi-4"
 
-# model_id = "Qwen/Qwen2-7B-Instruct"
+# # model_id = "Qwen/Qwen2-7B-Instruct"
 
 
-tokenizer = AutoTokenizer.from_pretrained(
-    # model_id
-    model_id,
-    # use_fast=False
-    token= token,
-trust_remote_code=True)
+# tokenizer = AutoTokenizer.from_pretrained(
+#     # model_id
+#     model_id,
+#     # use_fast=False
+#     token= token,
+# trust_remote_code=True)
 
 
-accelerator = Accelerator()
+# accelerator = Accelerator()
 
-model = AutoModelForCausalLM.from_pretrained(model_id, token= token, 
-                                                 # torch_dtype= torch.uint8, 
-                                             torch_dtype=torch.bfloat16,
-                                              # load_in_8bit=True,
-                                             # #  # torch_dtype=torch.fl,
-                                             attn_implementation="flash_attention_2",
-                                             low_cpu_mem_usage=True,
-                                             trust_remote_code=True,
-                                             device_map='cuda',
-                                             # device_map=accelerator.device_map,
+# model = AutoModelForCausalLM.from_pretrained(model_id, token= token, 
+#                                                  # torch_dtype= torch.uint8, 
+#                                              torch_dtype=torch.bfloat16,
+#                                               # load_in_8bit=True,
+#                                              # #  # torch_dtype=torch.fl,
+#                                              attn_implementation="flash_attention_2",
+#                                              low_cpu_mem_usage=True,
+#                                              trust_remote_code=True,
+#                                              device_map='cuda',
+#                                              # device_map=accelerator.device_map,
                                              
-                                            )
+#                                             )
 
 
 
 
 
-# 
-model = accelerator.prepare(model)
-from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+# # 
+# model = accelerator.prepare(model)
+# from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
 
-pipe = pipeline(
+# pipe = pipeline(
+#     "text-generation",
+#     model=model,
+#     tokenizer=tokenizer,
+# )
+
+
+
+
+pipeline = transformers.pipeline(
     "text-generation",
-    model=model,
-    tokenizer=tokenizer,
+    model="microsoft/phi-4",
+    model_kwargs={"torch_dtype": "auto"},
+    device_map="auto",
 )