many updates

app.py

@@ -1,5 +1,7 @@
 import gradio as gr
-from huggingface_hub import InferenceClient
+from transformers import AutoTokenizer, AutoModelForCausalLM, TextIteratorStreamer
+import torch
+from threading import Thread
 
 # Fixed system prompt (your "persona")
 SYSTEM_PROMPT = (
@@ -9,43 +11,90 @@ SYSTEM_PROMPT = (
     "developer tooling. You answer interview questions clearly, professionally, and naturally."
 )
 
+# Load model and tokenizer
+print("Loading model...")
+model_name = "Molchevsky/ai_resume_llama-3.2-3b"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(
+    model_name,
+    torch_dtype=torch.float16,
+    device_map="auto",
+    low_cpu_mem_usage=True,
+    trust_remote_code=True
+)
+print("Model loaded successfully!")
+
 def respond(
     message,
     history: list[dict[str, str]],
     max_tokens,
     temperature,
     top_p,
-    hf_token: gr.OAuthToken,
 ):
     """
-    Sends a chat request to your model hosted on Hugging Face.
+    Generate response using the local model.
     """
-    client = InferenceClient(
-        token=hf_token.token,
-        model="Molchevsky/ai_resume_llama-3.2-3b"   # <--- your model here
-    )
-
-    # Compose chat history with system prompt
+    # Build conversation with system prompt
     messages = [{"role": "system", "content": SYSTEM_PROMPT}]
     messages.extend(history)
     messages.append({"role": "user", "content": message})
-
+    
+    # Apply chat template if available
+    try:
+        formatted_chat = tokenizer.apply_chat_template(
+            messages,
+            tokenize=False,
+            add_generation_prompt=True
+        )
+    except Exception:
+        # Fallback formatting if chat template fails
+        formatted_chat = ""
+        for msg in messages:
+            if msg["role"] == "system":
+                formatted_chat += f"System: {msg['content']}\n\n"
+            elif msg["role"] == "user":
+                formatted_chat += f"User: {msg['content']}\n"
+            elif msg["role"] == "assistant":
+                formatted_chat += f"Assistant: {msg['content']}\n"
+        formatted_chat += "Assistant: "
+    
+    # Tokenize
+    inputs = tokenizer(formatted_chat, return_tensors="pt").to(model.device)
+    
+    # Set up streamer for real-time output
+    streamer = TextIteratorStreamer(
+        tokenizer,
+        timeout=60,
+        skip_prompt=True,
+        skip_special_tokens=True
+    )
+    
+    # Generation parameters
+    generation_kwargs = {
+        **inputs,
+        "streamer": streamer,
+        "max_new_tokens": max_tokens,
+        "temperature": temperature,
+        "top_p": top_p,
+        "do_sample": True,
+        "pad_token_id": tokenizer.eos_token_id,
+        "eos_token_id": tokenizer.eos_token_id,
+    }
+    
+    # Start generation in separate thread
+    generation_thread = Thread(target=model.generate, kwargs=generation_kwargs)
+    generation_thread.start()
+    
+    # Stream the response
     response = ""
-    for msg in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        choices = msg.choices
-        token = ""
-        if len(choices) and choices[0].delta.content:
-            token = choices[0].delta.content
-        response += token
-        yield response
-
+    try:
+        for token in streamer:
+            response += token
+            yield response
+    except Exception as e:
+        yield f"Error generating response: {str(e)}"
 
+# Create the chat interface
 chatbot = gr.ChatInterface(
     respond,
     type="messages",
@@ -63,10 +112,7 @@ chatbot = gr.ChatInterface(
 )
 
 with gr.Blocks() as demo:
-    with gr.Sidebar():
-        gr.LoginButton()
     chatbot.render()
 
-
 if __name__ == "__main__":
-    demo.launch(debug=True)
+    demo.launch(debug=True)

requirements.txt

@@ -1,4 +1,5 @@
 gradio
-torch
 transformers
-accelerate
+torch
+accelerate
+safetensors