many updates

app.py

@@ -1,7 +1,6 @@
 import gradio as gr
-from transformers import AutoTokenizer, AutoModelForCausalLM, TextIteratorStreamer
+from transformers import AutoTokenizer, AutoModelForCausalLM
 import torch
-from threading import Thread
 import traceback
 
 # Fixed system prompt (your "persona")
@@ -32,8 +31,8 @@ def load_model():
         # CPU-optimized model loading
         model = AutoModelForCausalLM.from_pretrained(
             model_name,
-            torch_dtype=torch.float32,  # Use float32 for CPU
-            device_map=None,  # Don't use device_map for CPU
+            torch_dtype=torch.float32,
+            device_map=None,
             low_cpu_mem_usage=True,
             trust_remote_code=True,
             use_cache=True,
@@ -41,11 +40,9 @@ def load_model():
         
         # Explicitly move to CPU
         model = model.to('cpu')
+        model.eval()  # Set to evaluation mode
         
         print(f"Model loaded successfully on CPU!")
-        print(f"Model device: {next(model.parameters()).device}")
-        print(f"Model dtype: {next(model.parameters()).dtype}")
-        
         return True
         
     except Exception as e:
@@ -56,121 +53,78 @@ def load_model():
 # Load model at startup
 model_loaded = load_model()
 
-def respond(
-    message,
-    history: list[dict[str, str]],
-    max_tokens,
-    temperature,
-    top_p,
-):
+def simple_respond(message, history, max_tokens, temperature, top_p):
     """
-    Generate response using CPU inference.
+    Simple non-streaming generation for debugging.
     """
     if not model_loaded or model is None or tokenizer is None:
-        yield "Error: Model not loaded properly. Please check the logs."
-        return
+        return "Error: Model not loaded properly."
     
     try:
-        print(f"Processing message: {message}")
-        
-        # Keep conversation history manageable for CPU
-        recent_history = history[-3:] if len(history) > 3 else history
-        
-        # Build simple conversation format
-        conversation_text = f"{SYSTEM_PROMPT}\n\n"
-        
-        # Add recent history
-        for msg in recent_history:
-            if msg.get("role") == "user":
-                conversation_text += f"User: {msg['content']}\n"
-            elif msg.get("role") == "assistant":
-                conversation_text += f"Assistant: {msg['content']}\n"
+        print(f"Processing: {message}")
         
-        conversation_text += f"User: {message}\nAssistant:"
+        # Very simple prompt
+        prompt = f"User: {message}\nAssistant:"
+        print(f"Prompt: {repr(prompt)}")
         
-        print(f"Prompt length: {len(conversation_text)}")
+        # Tokenize
+        inputs = tokenizer(prompt, return_tensors="pt", truncation=True, max_length=512)
+        print(f"Input shape: {inputs.input_ids.shape}")
+        print(f"Input tokens: {inputs.input_ids[0][:10]}")  # First 10 tokens
         
-        # Tokenize - keep it simple for CPU
-        inputs = tokenizer(
-            conversation_text, 
-            return_tensors="pt",
-            truncation=True,
-            max_length=1024,  # Shorter for CPU
-            padding=False
-        )
-        
-        print(f"Input tokens shape: {inputs.input_ids.shape}")
-        
-        # CPU-optimized generation with streaming
-        streamer = TextIteratorStreamer(
-            tokenizer,
-            timeout=120,  # Longer timeout for CPU
-            skip_prompt=True,
-            skip_special_tokens=True
-        )
+        # Simple generation - no streaming
+        print("Starting generation...")
+        with torch.no_grad():
+            outputs = model.generate(
+                inputs.input_ids,
+                attention_mask=inputs.attention_mask,
+                max_new_tokens=20,  # Very small for testing
+                temperature=0.7,
+                do_sample=True,
+                pad_token_id=tokenizer.eos_token_id,
+                eos_token_id=tokenizer.eos_token_id,
+            )
         
-        generation_kwargs = {
-            "input_ids": inputs.input_ids,
-            "attention_mask": inputs.attention_mask,
-            "streamer": streamer,
-            "max_new_tokens": min(max_tokens, 200),  # Limit for CPU
-            "temperature": temperature,
-            "top_p": top_p,
-            "do_sample": True,
-            "pad_token_id": tokenizer.eos_token_id,
-            "eos_token_id": tokenizer.eos_token_id,
-            "use_cache": True,
-            # CPU-specific optimizations
-            "num_beams": 1,  # No beam search for speed
-        }
+        print("Generation completed!")
+        print(f"Output shape: {outputs.shape}")
         
-        print("Starting CPU generation...")
+        # Decode only the new tokens
+        new_tokens = outputs[0][inputs.input_ids.shape[1]:]
+        response = tokenizer.decode(new_tokens, skip_special_tokens=True)
         
-        # Start generation in thread
-        generation_thread = Thread(target=model.generate, kwargs=generation_kwargs)
-        generation_thread.start()
+        print(f"Response: {repr(response)}")
         
-        # Stream response
-        response = ""
-        token_count = 0
+        if not response.strip():
+            return "Model generated empty response. This might be a model configuration issue."
         
-        for token in streamer:
-            response += token
-            token_count += 1
-            
-            # Yield periodically for better UX
-            if token_count % 5 == 0 or len(response) > len(response.split()[-1]):
-                yield response
-        
-        # Final yield
-        yield response
-        print(f"Generation completed. Response length: {len(response)}")
+        return response.strip()
         
     except Exception as e:
-        error_msg = f"Error in generation: {str(e)}"
+        error_msg = f"Error: {str(e)}"
         print(error_msg)
         print(traceback.format_exc())
-        yield error_msg
-
-# Create the chat interface
-chatbot = gr.ChatInterface(
-    respond,
-    type="messages",
-    additional_inputs=[
-        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p (nucleus sampling)",
-        ),
-    ],
-)
+        return error_msg
 
+# Create simple interface for testing
 with gr.Blocks() as demo:
-    chatbot.render()
+    gr.Markdown("# Debug Version - Simple Generation Test")
+    
+    with gr.Row():
+        msg_input = gr.Textbox(label="Message", placeholder="Type your message...")
+        send_btn = gr.Button("Send")
+    
+    output = gr.Textbox(label="Response", lines=5)
+    
+    # Simple controls
+    max_tokens = gr.Slider(1, 100, value=20, label="Max Tokens")
+    temperature = gr.Slider(0.1, 2.0, value=0.7, label="Temperature")
+    top_p = gr.Slider(0.1, 1.0, value=0.9, label="Top-p")
+    
+    send_btn.click(
+        simple_respond,
+        inputs=[msg_input, gr.State([]), max_tokens, temperature, top_p],
+        outputs=output
+    )
 
 if __name__ == "__main__":
     demo.launch(debug=True)