File size: 9,109 Bytes
93ed498 e2ec341 f4903ba 0d02f18 addc716 93ed498 b43bcfd 17749ab 9d23d35 e48da0f 265da87 93ed498 16e49b4 4ad0753 e18bb87 4ad0753 17749ab 58a3a72 7b32bf2 16e49b4 58a3a72 16e49b4 5097018 6390b56 06cc7e1 b43bcfd 6390b56 ec7e05a 9bfe2be ec7e05a 84d8cd6 d1b5796 9d23d35 5097018 9d23d35 d1b5796 9d23d35 29e7041 3ae4a47 9d23d35 d491d34 9d23d35 0eaea57 9d23d35 29e7041 9b28aea 9d23d35 9b28aea 9d23d35 9bfe2be 861a731 b5411ca 195b309 d491d34 4ad0753 3ae4a47 2d9088a 0d02f18 f427724 79de8f6 9d23d35 add0b80 9d23d35 f427724 9d23d35 bc7e35a 1b0ef2e 9d23d35 6d1ecc6 9d23d35 5554587 4ad0753 195b309 3446fdb 93ed498 9d23d35 4935248 9d23d35 4935248 6b6ca97 4935248 9d23d35 07a526e 9d23d35 993a41a 9d23d35 a33cd2c 993a41a ce6dc13 84e8a41 586b115 9d23d35 5554587 9d23d35 a3c3c74 9d23d35 ab6fbd7 b5411ca 2cb2161 d1b5796 84d8cd6 195b309 84d8cd6 ec7e05a 84d8cd6 6390b56 93ed498 6390b56 93ed498 6390b56 93ed498 6390b56 93ed498 6390b56 93ed498 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 |
import gradio as gr
import spaces
import torch
from torch.cuda.amp import autocast
import subprocess
from huggingface_hub import InferenceClient
import os
import psutil
import json
import subprocess
from threading import Thread
import torch
import spaces
import gradio as gr
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, TextIteratorStreamer
subprocess.run('pip install flash-attn --no-build-isolation', env={'FLASH_ATTENTION_SKIP_CUDA_BUILD': "TRUE"}, shell=True)
from transformers import AutoConfig, AutoModel
"""
For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
"""
from accelerate import init_empty_weights, infer_auto_device_map, load_checkpoint_and_dispatch
from accelerate import Accelerator
subprocess.run(
"pip install psutil",
shell=True,
)
import bitsandbytes as bnb # Import bitsandbytes for 8-bit quantization
from datetime import datetime
subprocess.run(
"pip install flash-attn --no-build-isolation",
env={"FLASH_ATTENTION_SKIP_CUDA_BUILD": "TRUE"},
shell=True,
)
# client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
# pip install 'git+https://github.com/huggingface/transformers.git'
token=os.getenv('token')
print('token = ',token)
from transformers import AutoModelForCausalLM, AutoTokenizer
import transformers
# model_id = "mistralai/Mistral-7B-v0.3"
# model_id = "microsoft/Phi-3-medium-4k-instruct"
# # model_id = "microsoft/phi-4"
# # model_id = "Qwen/Qwen2-7B-Instruct"
# tokenizer = AutoTokenizer.from_pretrained(
# # model_id
# model_id,
# # use_fast=False
# token= token,
# trust_remote_code=True)
# accelerator = Accelerator()
# model = AutoModelForCausalLM.from_pretrained(model_id, token= token,
# # torch_dtype= torch.uint8,
# torch_dtype=torch.bfloat16,
# # load_in_8bit=True,
# # # # torch_dtype=torch.fl,
# attn_implementation="flash_attention_2",
# low_cpu_mem_usage=True,
# trust_remote_code=True,
# device_map='cuda',
# # device_map=accelerator.device_map,
# )
# #
# model = accelerator.prepare(model)
# from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
# pipe = pipeline(
# "text-generation",
# model=model,
# tokenizer=tokenizer,
# )
# pipeline = transformers.pipeline(
# "text-generation",
# model="microsoft/phi-4",
# model_kwargs={"torch_dtype": "auto"},
# device_map="auto",
# )
# device_map = infer_auto_device_map(model, max_memory={0: "79GB", "cpu":"65GB" })
# Load the model with the inferred device map
# model = load_checkpoint_and_dispatch(model, model_id, device_map=device_map, no_split_module_classes=["GPTJBlock"])
# model.half()
# MODEL_ID = "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
MODEL_ID = "microsoft/phi-4"
CHAT_TEMPLATE = "َAuto"
MODEL_NAME = MODEL_ID.split("/")[-1]
CONTEXT_LENGTH = 16000
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
quantization_config = BitsAndBytesConfig(
load_in_4bit=True,
bnb_4bit_compute_dtype=torch.bfloat16
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
model = AutoModelForCausalLM.from_pretrained(
MODEL_ID,
device_map="auto",
low_cpu_mem_usage=True,
torch_dtype=torch.bfloat16,
# quantization_config=quantization_config,
attn_implementation="flash_attention_2",
)
accelerator = Accelerator()
model = accelerator.prepare(model)
import json
def str_to_json(str_obj):
json_obj = json.loads(str_obj)
return json_obj
@spaces.GPU(duration=60)
def respond(
message,
history: list[tuple[str, str]],
system_message,
max_tokens,
temperature,
top_p):
messages = []
json_obj = str_to_json(message)
print(json_obj)
messages= json_obj
stop_tokens = ["<|endoftext|>", "<|im_end|>"]
instruction ="" #'<|im_start|>system\n' + system_message + '\n<|im_end|>\n'
for qq in messages:
role= qq['role']
content= qq['content']
instruction+= f'<|im_start|>{role}<|im_sep|>\n{content}\n<|im_end|>\n'
instruction+='<|im_start|>assistant<|im_sep|>\n'
# for user, assistant in history:
# instruction += f'<|im_start|>user\n{user}\n<|im_end|>\n<|im_start|>assistant\n{assistant}\n<|im_end|>\n'
# instruction += f'<|im_start|>user\n{message}\n<|im_end|>\n<|im_start|>assistant\n'
print(instruction)
streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
enc = tokenizer(instruction, return_tensors="pt", padding=True, truncation=True)
input_ids, attention_mask = enc.input_ids, enc.attention_mask
if input_ids.shape[1] > CONTEXT_LENGTH:
input_ids = input_ids[:, -CONTEXT_LENGTH:]
attention_mask = attention_mask[:, -CONTEXT_LENGTH:]
generate_kwargs = dict(
input_ids=input_ids.to(device),
attention_mask=attention_mask.to(device),
streamer=streamer,
do_sample=True,
temperature=temperature,
max_new_tokens=max_tokens,
top_k=40,
repetition_penalty=1.1,
top_p=0.95
)
t = Thread(target=model.generate, kwargs=generate_kwargs)
t.start()
outputs= ""
for new_token in streamer:
print(new_token," ")
outputs = outputs+ new_token
print("output ",outputs)
yield outputs
# yield 'retuend'
# model.to(accelerator.device)
# messages = []
# json_obj = str_to_json(message)
# print(json_obj)
# messages= json_obj
# # input_ids = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt").to(accelerator.device)
# # input_ids2 = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, return_tensors="pt") #.to('cuda')
# # print(f"Converted input_ids dtype: {input_ids.dtype}")
# # input_str= str(input_ids2)
# # print('input str = ', input_str)
# generation_args = {
# "max_new_tokens": max_tokens,
# "return_full_text": False,
# "temperature": temperature,
# "do_sample": False,
# }
# output = pipe(messages, **generation_args)
# print(output[0]['generated_text'])
# gen_text=output[0]['generated_text']
# # with torch.no_grad():
# # gen_tokens = model.generate(
# # input_ids,
# # max_new_tokens=max_tokens,
# # # do_sample=True,
# # temperature=temperature,
# # )
# # gen_text = tokenizer.decode(gen_tokens[0])
# # print(gen_text)
# # gen_text= gen_text.replace(input_str,'')
# # gen_text= gen_text.replace('<|im_end|>','')
# yield gen_text
# messages = [
# # {"role": "user", "content": "What is your favourite condiment?"},
# # {"role": "assistant", "content": "Well, I'm quite partial to a good squeeze of fresh lemon juice. It adds just the right amount of zesty flavour to whatever I'm cooking up in the kitchen!"},
# # {"role": "user", "content": "Do you have mayonnaise recipes?"}
# ]
# inputs = tokenizer.apply_chat_template(messages, return_tensors="pt").to("cuda")
# outputs = model.generate(inputs, max_new_tokens=2000)
# gen_text=tokenizer.decode(outputs[0], skip_special_tokens=True)
# print(gen_text)
# yield gen_text
# for val in history:
# if val[0]:
# messages.append({"role": "user", "content": val[0]})
# if val[1]:
# messages.append({"role": "assistant", "content": val[1]})
# messages.append({"role": "user", "content": message})
# response = ""
# for message in client.chat_completion(
# messages,
# max_tokens=max_tokens,
# stream=True,
# temperature=temperature,
# top_p=top_p,
# ):
# token = message.choices[0].delta.content
# response += token
# yield response
"""
For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
"""
demo = gr.ChatInterface(
respond,
additional_inputs=[
gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
gr.Slider(
minimum=0.1,
maximum=1.0,
value=0.95,
step=0.05,
label="Top-p (nucleus sampling)",
),
],
)
if __name__ == "__main__":
demo.launch() |