File size: 9,109 Bytes
93ed498
e2ec341
f4903ba
0d02f18
addc716
93ed498
b43bcfd
17749ab
 
9d23d35
 
 
 
 
 
 
 
 
 
e48da0f
265da87
 
93ed498
 
 
16e49b4
4ad0753
e18bb87
 
4ad0753
17749ab
 
 
 
 
58a3a72
7b32bf2
 
 
16e49b4
58a3a72
 
 
16e49b4
 
 
 
 
 
5097018
6390b56
 
06cc7e1
 
b43bcfd
 
6390b56
ec7e05a
9bfe2be
ec7e05a
84d8cd6
d1b5796
9d23d35
 
5097018
9d23d35
d1b5796
 
9d23d35
 
 
 
 
 
29e7041
3ae4a47
9d23d35
d491d34
9d23d35
 
 
 
 
 
 
 
 
 
0eaea57
9d23d35
29e7041
 
9b28aea
 
 
9d23d35
 
 
9b28aea
9d23d35
 
 
 
 
9bfe2be
 
 
 
861a731
 
 
 
 
 
b5411ca
195b309
d491d34
4ad0753
 
3ae4a47
2d9088a
0d02f18
f427724
 
79de8f6
9d23d35
 
 
 
 
 
 
add0b80
9d23d35
 
 
f427724
9d23d35
 
bc7e35a
1b0ef2e
 
9d23d35
 
 
6d1ecc6
 
 
 
 
9d23d35
 
5554587
 
 
 
 
4ad0753
195b309
3446fdb
93ed498
 
 
 
 
 
9d23d35
4935248
 
 
 
 
 
 
9d23d35
 
4935248
 
 
 
 
6b6ca97
 
4935248
 
 
 
9d23d35
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
07a526e
 
 
 
9d23d35
 
 
993a41a
9d23d35
a33cd2c
993a41a
 
 
 
ce6dc13
84e8a41
586b115
 
9d23d35
 
 
5554587
9d23d35
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a3c3c74
9d23d35
ab6fbd7
b5411ca
2cb2161
 
 
 
 
d1b5796
84d8cd6
195b309
84d8cd6
 
ec7e05a
84d8cd6
 
6390b56
 
 
 
 
93ed498
6390b56
93ed498
6390b56
93ed498
6390b56
 
 
 
 
 
 
 
93ed498
6390b56
 
93ed498
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
import gradio as gr
import spaces
import torch
from torch.cuda.amp import autocast
import subprocess
from huggingface_hub import InferenceClient
import os
import psutil

import json
import subprocess
from threading import Thread

import torch
import spaces
import gradio as gr
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, TextIteratorStreamer

subprocess.run('pip install flash-attn --no-build-isolation', env={'FLASH_ATTENTION_SKIP_CUDA_BUILD': "TRUE"}, shell=True)
from transformers import AutoConfig, AutoModel


"""
For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
"""

from accelerate import init_empty_weights, infer_auto_device_map, load_checkpoint_and_dispatch
from accelerate import Accelerator


subprocess.run(
    "pip install psutil",
   
    shell=True,
)

import bitsandbytes as bnb  # Import bitsandbytes for 8-bit quantization



from datetime import datetime


subprocess.run(
    "pip install flash-attn --no-build-isolation",
    env={"FLASH_ATTENTION_SKIP_CUDA_BUILD": "TRUE"},
    shell=True,
)

# client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
# pip install 'git+https://github.com/huggingface/transformers.git'



token=os.getenv('token')
print('token = ',token)

from transformers import AutoModelForCausalLM, AutoTokenizer
import transformers

# model_id = "mistralai/Mistral-7B-v0.3"

# model_id = "microsoft/Phi-3-medium-4k-instruct"
# # model_id = "microsoft/phi-4"

# # model_id = "Qwen/Qwen2-7B-Instruct"


# tokenizer = AutoTokenizer.from_pretrained(
#     # model_id
#     model_id,
#     # use_fast=False
#     token= token,
# trust_remote_code=True)


# accelerator = Accelerator()

# model = AutoModelForCausalLM.from_pretrained(model_id, token= token, 
#                                                  # torch_dtype= torch.uint8, 
#                                              torch_dtype=torch.bfloat16,
#                                               # load_in_8bit=True,
#                                              # #  # torch_dtype=torch.fl,
#                                              attn_implementation="flash_attention_2",
#                                              low_cpu_mem_usage=True,
#                                              trust_remote_code=True,
#                                              device_map='cuda',
#                                              # device_map=accelerator.device_map,
                                             
#                                             )





# # 
# model = accelerator.prepare(model)
# from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline

# pipe = pipeline(
#     "text-generation",
#     model=model,
#     tokenizer=tokenizer,
# )




# pipeline = transformers.pipeline(
#     "text-generation",
#     model="microsoft/phi-4",
#     model_kwargs={"torch_dtype": "auto"},
#     device_map="auto",
# )


# device_map = infer_auto_device_map(model, max_memory={0: "79GB", "cpu":"65GB" })

# Load the model with the inferred device map
# model = load_checkpoint_and_dispatch(model, model_id, device_map=device_map, no_split_module_classes=["GPTJBlock"])
# model.half()

# MODEL_ID = "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
MODEL_ID = "microsoft/phi-4"

CHAT_TEMPLATE = "َAuto"
MODEL_NAME = MODEL_ID.split("/")[-1]
CONTEXT_LENGTH = 16000


device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
quantization_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_compute_dtype=torch.bfloat16
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID,
    device_map="auto",
     low_cpu_mem_usage=True,
    torch_dtype=torch.bfloat16,
    # quantization_config=quantization_config,
    attn_implementation="flash_attention_2",
)

accelerator = Accelerator()

model = accelerator.prepare(model)




import json

def str_to_json(str_obj):
    json_obj = json.loads(str_obj)
    return json_obj


@spaces.GPU(duration=60)
def respond(
    message,
    history: list[tuple[str, str]],
    system_message,
    max_tokens,
    temperature,
    top_p):

    messages = []
    json_obj = str_to_json(message)
    print(json_obj)
    
    messages= json_obj

    
    stop_tokens = ["<|endoftext|>", "<|im_end|>"]
    instruction ="" #'<|im_start|>system\n' + system_message + '\n<|im_end|>\n'
    for qq in messages:
        role= qq['role']
        content= qq['content']
        instruction+= f'<|im_start|>{role}<|im_sep|>\n{content}\n<|im_end|>\n'

    instruction+='<|im_start|>assistant<|im_sep|>\n'
        
    # for user, assistant in history:
    #     instruction += f'<|im_start|>user\n{user}\n<|im_end|>\n<|im_start|>assistant\n{assistant}\n<|im_end|>\n'
    # instruction += f'<|im_start|>user\n{message}\n<|im_end|>\n<|im_start|>assistant\n'
    
    print(instruction)
    
    streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
    enc = tokenizer(instruction, return_tensors="pt", padding=True, truncation=True)
    input_ids, attention_mask = enc.input_ids, enc.attention_mask

    if input_ids.shape[1] > CONTEXT_LENGTH:
        input_ids = input_ids[:, -CONTEXT_LENGTH:]
        attention_mask = attention_mask[:, -CONTEXT_LENGTH:]

    generate_kwargs = dict(
        input_ids=input_ids.to(device),
        attention_mask=attention_mask.to(device),
        streamer=streamer,
        do_sample=True,
        temperature=temperature,
        max_new_tokens=max_tokens,
        top_k=40,
        repetition_penalty=1.1,
        top_p=0.95
    )
    t = Thread(target=model.generate, kwargs=generate_kwargs)
    t.start()
    outputs= ""
    for new_token in streamer:
        print(new_token," ")
        outputs = outputs+  new_token

    print("output ",outputs)
    yield outputs
    
    # yield 'retuend'
    # model.to(accelerator.device)

#     messages = []
#     json_obj = str_to_json(message)
#     print(json_obj)
    
#     messages= json_obj

#     # input_ids = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt").to(accelerator.device)
#     # input_ids2 = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, return_tensors="pt") #.to('cuda')
#     # print(f"Converted input_ids dtype: {input_ids.dtype}")
#     # input_str= str(input_ids2)
#     # print('input str = ', input_str)

#     generation_args = {
#     "max_new_tokens": max_tokens,
#     "return_full_text": False,
#     "temperature": temperature,
#     "do_sample": False,
# }

#     output = pipe(messages, **generation_args)
#     print(output[0]['generated_text'])
#     gen_text=output[0]['generated_text']

#     # with torch.no_grad():
#     #     gen_tokens = model.generate(
#     # input_ids, 
#     # max_new_tokens=max_tokens, 
#     # # do_sample=True, 
#     # temperature=temperature,
#     # )

#     # gen_text = tokenizer.decode(gen_tokens[0])
#     # print(gen_text)
#     # gen_text= gen_text.replace(input_str,'')
#     # gen_text= gen_text.replace('<|im_end|>','')
    
#     yield gen_text
   
  
#     messages = [
#     # {"role": "user", "content": "What is your favourite condiment?"},
#     # {"role": "assistant", "content": "Well, I'm quite partial to a good squeeze of fresh lemon juice. It adds just the right amount of zesty flavour to whatever I'm cooking up in the kitchen!"},
#     # {"role": "user", "content": "Do you have mayonnaise recipes?"}
# ]

    # inputs = tokenizer.apply_chat_template(messages, return_tensors="pt").to("cuda")

    # outputs = model.generate(inputs, max_new_tokens=2000)
    # gen_text=tokenizer.decode(outputs[0], skip_special_tokens=True)
   
    # print(gen_text)
    # yield gen_text
    # for val in history:
    #     if val[0]:
    #         messages.append({"role": "user", "content": val[0]})
    #     if val[1]:
    #         messages.append({"role": "assistant", "content": val[1]})

    # messages.append({"role": "user", "content": message})

    # response = ""

    # for message in client.chat_completion(
    #     messages,
    #     max_tokens=max_tokens,
    #     stream=True,
    #     temperature=temperature,
    #     top_p=top_p,
    # ):
    #     token = message.choices[0].delta.content

    #     response += token
    #     yield response

"""
For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
"""
demo = gr.ChatInterface(
    respond,
    additional_inputs=[
        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
        gr.Slider(
            minimum=0.1,
            maximum=1.0,
            value=0.95,
            step=0.05,
            label="Top-p (nucleus sampling)",
        ),
    ],
)


if __name__ == "__main__":
    demo.launch()