chatbot add

.gitignore

@@ -9,6 +9,8 @@ env/
 venv/
 myenv/
 .myenv/
+myenv31/
+.myenv31/
 *.env
 
 # Jupyter/IPython

app/agent/custom_chatbot.py

@@ -1,10 +1,11 @@
 import os
 import logging
 from fastapi import APIRouter, HTTPException
-from pydantic.v1 import BaseModel, EmailStr
+from pydantic.v1 import BaseModel
 from dotenv import load_dotenv
 
 from langchain_groq import ChatGroq
+
 from langchain_openai import OpenAIEmbeddings
 from langchain_community.vectorstores import FAISS
 from langchain.text_splitter import RecursiveCharacterTextSplitter
@@ -15,15 +16,26 @@ from langchain.chains import create_retrieval_chain
 # Load environment variables
 load_dotenv()
 
+# Router and logger
 router = APIRouter()
-
 logger = logging.getLogger("custom_chatbot")
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
 
-# LangChain LLM setup
+# Validate GROQ API Key
 groq_api_key = os.getenv("GROQ_API_KEY")
-llm = ChatGroq(groq_api_key=groq_api_key, model_name="Llama3-8b-8192")
-
-# Prompt template
+if not groq_api_key:
+    logger.error("❌ GROQ_API_KEY is not set in the environment.")
+    raise RuntimeError("GROQ_API_KEY must be set in .env or environment variables.")
+
+# LLM Initialization
+try:
+    llm = ChatGroq(groq_api_key=groq_api_key, model_name="Llama3-8b-8192")
+    logger.info("✅ ChatGroq LLM initialized successfully.")
+except Exception as e:
+    logger.exception("❌ Failed to initialize ChatGroq LLM.")
+    raise
+
+# Prompt Template
 prompt_template = ChatPromptTemplate.from_template("""
 Answer the question based only on the provided context.
 <context>
@@ -33,27 +45,37 @@ Answer the question based only on the provided context.
 Question: {input}
 """)
 
-# Input schema with user_id
+# Input schema
 class ChatRequest(BaseModel):
     query: str
     user_id: int
 
-# Load vector store for a specific user
+# Load vector store for a given user
 def load_user_vector_store(user_id: int):
     user_path = f"vector_store/user_{user_id}"
     index_file = os.path.join(user_path, "index.faiss")
 
+    logger.info(f"🔍 Looking for vector store at {index_file}")
     if not os.path.exists(index_file):
-        raise FileNotFoundError(f"No vector store found for user {user_id}")
+        msg = f"No vector store found for user {user_id}"
+        logger.warning(f"📭 {msg}")
+        raise FileNotFoundError(msg)
 
-    embeddings = OpenAIEmbeddings()
-    return FAISS.load_local(user_path, embeddings)
+    try:
+        embeddings = OpenAIEmbeddings()
+        vector_store = FAISS.load_local(user_path, embeddings,allow_dangerous_deserialization=True)
+        logger.info(f"📦 Vector store loaded for user {user_id}")
+        return vector_store
+    except Exception as e:
+        logger.exception(f"❌ Failed to load vector store for user {user_id}")
+        raise
 
-# Endpoint
+# Main chatbot endpoint
 @router.post("/custom-chatbot")
 async def custom_chatbot(request: ChatRequest):
     query = request.query
     user_id = request.user_id
+    logger.info(f"🤖 Received query from user {user_id}: {query}")
 
     try:
         vector_store = load_user_vector_store(user_id)
@@ -63,15 +85,17 @@ async def custom_chatbot(request: ChatRequest):
         rag_chain = create_retrieval_chain(retriever, doc_chain)
 
         response = rag_chain.invoke({"input": query})
+        logger.info(f"✅ Response generated for user {user_id}")
+
         return {
             "answer": response["answer"],
-            "sources": [doc.page_content for doc in response["context"]],
+            "sources": [doc.page_content for doc in response.get("context", [])],
         }
 
     except FileNotFoundError as e:
-        logger.warning(f"📭 Vector store missing for user {user_id}")
+        logger.warning(f"🚫 {e}")
         raise HTTPException(status_code=404, detail=str(e))
 
     except Exception as e:
-        logger.error(f"❌ Error in custom chatbot: {e}")
+        logger.exception("❌ Unexpected error in custom chatbot endpoint.")
         raise HTTPException(status_code=500, detail="Internal server error")

app/utils/whisper_llm.py

@@ -11,20 +11,19 @@ from langchain_openai import OpenAIEmbeddings
 from langchain_core.documents import Document
 from langchain_community.vectorstores import FAISS
 
-from app.db import SessionLocal  # Assuming SQLAlchemy session
-from app.models import User  # Assuming SQLAlchemy User model
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+from app.models import User
 
 # Setup logger
 logger = logging.getLogger("app.utils.whisper_llm")
 logger.setLevel(logging.INFO)
-
 if not logger.handlers:
     handler = logging.StreamHandler()
     formatter = logging.Formatter("[%(asctime)s] %(levelname)s - %(message)s")
     handler.setFormatter(formatter)
     logger.addHandler(handler)
 
-
 # Whisper Model Initialization
 def get_whisper_model():
     if torch.cuda.is_available():
@@ -35,6 +34,7 @@ def get_whisper_model():
         device = "cpu"
         compute_type = "int8"
         logger.warning("⚠️ GPU not available: Falling back to CPU with int8 compute")
+
     try:
         model = WhisperModel("base", device=device, compute_type=compute_type)
         logger.info(f"📦 Loaded Faster-Whisper model on {device} with compute_type={compute_type}")
@@ -53,7 +53,6 @@ except Exception as e:
     logger.error(f"❌ Failed to load summarization pipeline: {e}")
     raise
 
-
 # Chunked summarization
 def summarize_in_chunks(text, chunk_size=800, overlap=100):
     summaries = []
@@ -71,25 +70,20 @@ def summarize_in_chunks(text, chunk_size=800, overlap=100):
             logger.error(f"❌ Chunk summarization failed: {e}")
     return " ".join(summaries)
 
+# Async user fetch using AsyncSession
+async def get_user(user_id: int, db: AsyncSession):
+    result = await db.execute(select(User).where(User.id == user_id))
+    return result.scalar_one_or_none()
 
-# 🧠 Get user from Neon DB
-def get_user(user_id: int):
-    db = SessionLocal()
-    try:
-        return db.query(User).filter(User.id == user_id).first()
-    finally:
-        db.close()
-
-
-# ⚡ Core Analyzer Function with per-user FAISS ingestion
-def analyze(video_url: str, user_id: int):
-    # Verify user exists
-    user = get_user(user_id)
+# 🧠 Core analyzer function with per-user FAISS ingestion
+async def analyze(video_url: str, user_id: int, db: AsyncSession):
+    user = await get_user(user_id, db)
     if not user:
-        raise ValueError(f"❌ User with ID {user_id} not found in Neon DB")
+        raise ValueError(f"❌ User with ID {user_id} not found in database.")
 
     logger.info(f"📥 Starting video analysis for user: {user.email} (ID: {user.id})")
 
+    # Step 1: Download video to temp file
     try:
         with tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") as tmp:
             with requests.get(video_url, stream=True, timeout=60) as response:
@@ -102,6 +96,7 @@ def analyze(video_url: str, user_id: int):
         logger.error(f"❌ Failed to download video: {e}")
         raise
 
+    # Step 2: Transcribe
     try:
         logger.info("🧠 Transcribing audio with Faster-Whisper...")
         segments, _ = whisper_model.transcribe(tmp_path)
@@ -111,6 +106,7 @@ def analyze(video_url: str, user_id: int):
         logger.error(f"❌ Transcription failed: {e}")
         raise
 
+    # Step 3: Summarize
     try:
         logger.info("📝 Summarizing transcript with Hugging Face model...")
         summary = summarize_in_chunks(text)
@@ -119,6 +115,7 @@ def analyze(video_url: str, user_id: int):
         logger.error(f"❌ Summarization failed: {e}")
         raise
 
+    # Step 4: Save to FAISS store
     try:
         logger.info("📊 Creating/updating FAISS vector store for user...")
         documents = [Document(page_content=summary)]

requirements.txt

@@ -1,47 +1,56 @@
+# Core
 fastapi
 uvicorn
 boto3
-python-multipart
-whisper
-transformers
-reportlab
 requests
-supabase
 python-dotenv
-pydantic[email]
-asyncpg 
-sqlalchemy 
-databases 
-psycopg2-binary 
+python-multipart
+pydantic[email]>=1.10,<2.0  # V1 for compatibility with many frameworks
+
+# DB
+asyncpg
+sqlalchemy>=2.0
+databases
+psycopg2-binary
+
+# Auth
 passlib[bcrypt]
 python-jose[cryptography]
-faster-whisper
-torch==2.2.2+cu121
-torchvision==0.17.2+cu121
-torchaudio==2.2.2+cu121
---extra-index-url https://download.pytorch.org/whl/cu121
-# CTranslate2 GPU build for CUDA 12
-langchain_openai 
-langchain_core
-python-dotenv
-streamlit
-langchain_community
+
+# LLM & RAG
+langchain==0.1.13
+langchain-openai==0.1.7
+langchain-community==0.0.38
+langchain-core==0.1.53
+langchain-groq
+langchainhub
 langserve
-sse_starlette
-bs4
-pypdf
-chromadb
+langchain-objectbox
+
+# Embedding & vector DB
+sentence-transformers==2.2.2
 faiss-cpu
-groq
-cassio
+chromadb
+
+# Tools & Transcription
+transformers
+whisper
+faster-whisper==1.0.1
+ctranslate2>=4.0,<5
+PyPDF2
+pypdf
+reportlab
+bs4
 beautifulsoup4
-langchain-groq
+
+# Optional
+sse-starlette
 wikipedia
 arxiv
-langchainhub
-sentence_transformers
-PyPDF2
-langchain-objectbox
-pypdf
-langchain_groq
-langchain
+cassio
+streamlit
+
+# CUDA-enabled Torch (installed separately)
+# torch==2.2.2+cu121 and torchvision==0.17.2+cu121
+# Must be installed via pip with specific index:
+# pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121

setup-dubsway-env.bat

@@ -0,0 +1,81 @@
+@echo off
+setlocal enabledelayedexpansion
+
+REM ---- CONFIGURATION ----
+set ENV_NAME=myenv
+set PYTHON_EXE=python  REM Ensure python points to 3.10 or 3.11 in PATH
+set TORCH_INDEX=https://download.pytorch.org/whl/cu121
+
+echo.
+echo 🚀 Creating virtual environment: %ENV_NAME%
+%PYTHON_EXE% -m venv %ENV_NAME%
+
+echo.
+echo 🔄 Activating environment...
+call %ENV_NAME%\Scripts\activate.bat
+
+echo.
+echo 📦 Upgrading pip
+pip install --upgrade pip
+
+echo.
+echo 📄 Writing requirements.txt
+(
+echo fastapi
+echo uvicorn
+echo boto3
+echo requests
+echo python-dotenv
+echo python-multipart
+echo pydantic[email]>=1.10,<2.0
+echo asyncpg
+echo sqlalchemy>=2.0
+echo databases
+echo psycopg2-binary
+echo passlib[bcrypt]
+echo python-jose[cryptography]
+echo langchain==0.1.13
+echo langchain-openai==0.1.7
+echo langchain-community==0.0.38
+echo langchain-core==0.1.53
+echo langchain-groq==0.0.3
+echo langchainhub
+echo langserve
+echo langchain-objectbox
+echo sentence-transformers==2.2.2
+echo faiss-cpu
+echo chromadb
+echo transformers
+echo whisper
+echo faster-whisper==1.0.1
+echo ctranslate2==3.22.0
+echo PyPDF2
+echo pypdf
+echo reportlab
+echo bs4
+echo beautifulsoup4
+echo sse-starlette
+echo wikipedia
+echo arxiv
+echo cassio
+echo streamlit
+) > requirements.txt
+
+echo.
+echo 📥 Installing base packages...
+pip install -r requirements.txt
+
+echo.
+echo ⚡ Installing PyTorch (CUDA 12.1 build)...
+pip install torch torchvision torchaudio --index-url %TORCH_INDEX%
+
+echo.
+echo ✅ Verifying important packages...
+python -c "import torch; print('Torch version:', torch.__version__)"
+python -c "from langchain_groq import ChatGroq; print('✅ langchain_groq available')"
+python -c "from faster_whisper import WhisperModel; print('✅ faster-whisper ready')"
+python -c "import faiss; print('✅ faiss-cpu available')"
+
+echo.
+echo 🎉 Setup complete! Environment '%ENV_NAME%' is ready to use.
+endlocal

start-server.bat

@@ -0,0 +1,8 @@
+@echo off
+echo 🔁 Activating virtual environment...
+call .\myenv\Scripts\activate
+
+echo 🚀 Starting FastAPI server...
+python -m uvicorn app.main:app --reload
+
+pause

worker/daemon.py

@@ -2,6 +2,7 @@ import asyncio
 import os
 import time
 from datetime import datetime
+import traceback
 
 from sqlalchemy.future import select
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -10,7 +11,6 @@ from app.database import AsyncSessionLocal
 from app.models import VideoUpload
 from app.utils import whisper_llm, pdf, s3
 
-
 POLL_INTERVAL = 200  # seconds
 
 
@@ -26,33 +26,48 @@ async def process_pending_videos():
                 print(f"🎬 Processing video ID {video.id} for user {video.user_id}")
 
                 try:
-                    transcription, summary = whisper_llm.analyze(video.video_url)
+                    # ✅ New:
+                    transcription, summary = await whisper_llm.analyze(
+                        video_url=video.video_url,
+                        user_id=video.user_id,
+                        db=session  # passing the active AsyncSession
+                    )
+
                 except Exception as e:
-                    print(f"❌ Whisper failed: {e}")
+                    print(f"❌ Whisper failed for video {video.id}: {e}")
+                    traceback.print_exc()
                     continue
 
                 try:
                     pdf_bytes = pdf.generate(transcription, summary)
                 except Exception as e:
-                    print(f"❌ PDF generation failed: {e}")
+                    print(f"❌ PDF generation failed for video {video.id}: {e}")
+                    traceback.print_exc()
                     continue
 
                 try:
                     pdf_key = f"pdfs/{video.id}.pdf"
                     pdf_url = s3.upload_pdf_bytes(pdf_bytes, pdf_key)
                 except Exception as e:
-                    print(f"❌ Upload to S3 failed: {e}")
+                    print(f"❌ Upload to S3 failed for video {video.id}: {e}")
+                    traceback.print_exc()
                     continue
 
-                video.status = "completed"
-                video.pdf_url = pdf_url
-                video.updated_at = datetime.utcnow()
+                try:
+                    video.status = "completed"
+                    video.pdf_url = pdf_url
+                    video.updated_at = datetime.utcnow()
 
-                await session.commit()
-                print(f"✅ Completed video {video.id}")
+                    await session.commit()
+                    print(f"✅ Completed video {video.id}")
+
+                except Exception as e:
+                    print(f"❌ DB commit failed for video {video.id}: {e}")
+                    traceback.print_exc()
 
         except Exception as e:
             print(f"❌ DB error: {e}")
+            traceback.print_exc()
 
 
 async def run_worker():
@@ -63,6 +78,7 @@ async def run_worker():
             await process_pending_videos()
         except Exception as e:
             print(f"❌ Worker loop crashed: {e}")
+            traceback.print_exc()
         await asyncio.sleep(POLL_INTERVAL)