Update app.py

app.py

@@ -34,6 +34,7 @@ CHAT_VECTOR_MODE = "vector"
 CHAT_GRAPH_MODE = "graph"
 CHAT_FULLTEXT_MODE = "fulltext"
 CHAT_HYBRID_MODE = "hybrid"
+CHAT_COMPREHENSIVE_MODE = "comprehensive"  # Graph+Vector+Fulltext
 
 class Neo4jRAGSystem:
     def __init__(self, openai_api_key: str = None):
@@ -133,31 +134,43 @@ class Neo4jRAGSystem:
                 "document_filter": True,
                 "use_vector_search": True,
                 "use_graph_search": False,
-                "description": "Vector similarity search"
+                "use_fulltext_search": False,
+                "description": "Vector similarity search using embeddings"
             },
             CHAT_GRAPH_MODE: {
                 "mode": "graph",
                 "document_filter": False,
                 "use_vector_search": False,
                 "use_graph_search": True,
-                "description": "Graph-based knowledge retrieval"
+                "use_fulltext_search": False,
+                "description": "Graph-based knowledge retrieval using relationships"
             },
             CHAT_FULLTEXT_MODE: {
                 "mode": "fulltext",
                 "document_filter": True,
                 "use_vector_search": False,
                 "use_graph_search": False,
-                "description": "Full-text search"
+                "use_fulltext_search": True,
+                "description": "Full-text keyword search"
             },
             CHAT_HYBRID_MODE: {
                 "mode": "hybrid",
                 "document_filter": True,
                 "use_vector_search": True,
                 "use_graph_search": True,
+                "use_fulltext_search": False,
                 "description": "Combined vector and graph search"
+            },
+            CHAT_COMPREHENSIVE_MODE: {
+                "mode": "comprehensive",
+                "document_filter": True,
+                "use_vector_search": True,
+                "use_graph_search": True,
+                "use_fulltext_search": True,
+                "description": "Graph+Vector+Fulltext - Complete search using all methods"
             }
         }
-        return modes.get(mode, modes[CHAT_GRAPH_MODE])  # Default to graph mode if vector fails
+        return modes.get(mode, modes[CHAT_GRAPH_MODE])
 
     def create_neo4j_chat_message_history(self, session_id: str, write_access: bool = True):
         """Create Neo4j chat message history"""
@@ -228,22 +241,145 @@ class Neo4jRAGSystem:
                 "user": "chatbot"
             }
 
-    def fallback_search(self, question: str, document_names: List[str] = None):
-        """Fallback search using direct Neo4j queries"""
+    def fulltext_search(self, question: str, document_names: List[str] = None, limit: int = 5):
+        """Perform fulltext search on Neo4j"""
         try:
-            # Search for relevant nodes in the graph
+            # Create fulltext index if it doesn't exist
+            self.ensure_fulltext_index()
+            
+            # Build fulltext search query
+            if document_names:
+                search_query = """
+                CALL db.index.fulltext.queryNodes('fulltext_content', $question) 
+                YIELD node, score
+                WHERE any(doc IN $doc_names WHERE node.source CONTAINS doc)
+                RETURN node.content as content, node.text as text, node.title as title,
+                       node.source as source, score, labels(node) as labels
+                ORDER BY score DESC
+                LIMIT $limit
+                """
+                params = {"question": question, "doc_names": document_names, "limit": limit}
+            else:
+                search_query = """
+                CALL db.index.fulltext.queryNodes('fulltext_content', $question) 
+                YIELD node, score
+                RETURN node.content as content, node.text as text, node.title as title,
+                       node.source as source, score, labels(node) as labels
+                ORDER BY score DESC
+                LIMIT $limit
+                """
+                params = {"question": question, "limit": limit}
+            
+            results = self.graph.query(search_query, params)
+            
+            context_parts = []
+            sources = []
+            
+            for result in results:
+                content = result.get("content") or result.get("text") or ""
+                if content:
+                    context_parts.append(content)
+                    sources.append({
+                        "source": result.get("source", "Fulltext Search"),
+                        "title": result.get("title", "Unknown"),
+                        "score": result.get("score", 0),
+                        "search_type": "fulltext"
+                    })
+            
+            return "\n\n".join(context_parts), sources
+            
+        except Exception as e:
+            logger.warning(f"Fulltext search failed: {e}")
+            return "", []
+
+    def ensure_fulltext_index(self):
+        """Ensure fulltext index exists"""
+        try:
+            # Check if fulltext index exists
+            check_query = """
+            SHOW INDEXES YIELD name, type
+            WHERE name = 'fulltext_content' AND type = 'FULLTEXT'
+            RETURN count(*) as count
+            """
+            result = self.graph.query(check_query)
+            
+            if not result or result[0]["count"] == 0:
+                # Create fulltext index
+                create_index_query = """
+                CREATE FULLTEXT INDEX fulltext_content IF NOT EXISTS
+                FOR (n:Document|Chunk|Text) 
+                ON EACH [n.content, n.text, n.title]
+                """
+                self.graph.query(create_index_query)
+                logger.info("✅ Created fulltext index")
+            else:
+                logger.info("✅ Fulltext index already exists")
+                
+        except Exception as e:
+            logger.warning(f"Could not create fulltext index: {e}")
+
+    def vector_search(self, question: str, document_names: List[str] = None, k: int = 5):
+        """Perform vector similarity search"""
+        try:
+            if not self.vector_store:
+                return "", []
+            
+            search_kwargs = {"k": k}
+            if document_names:
+                search_kwargs["filter"] = {"document_name": {"$in": document_names}}
+            
+            retriever = self.vector_store.as_retriever(
+                search_type="similarity",
+                search_kwargs=search_kwargs
+            )
+            
+            relevant_docs = retriever.invoke(question)
+            
+            context = "\n\n".join([doc.page_content for doc in relevant_docs])
+            sources = [
+                {
+                    "source": doc.metadata.get("source", "Vector Search"),
+                    "content": doc.page_content[:200] + "..." if len(doc.page_content) > 200 else doc.page_content,
+                    "search_type": "vector"
+                }
+                for doc in relevant_docs
+            ]
+            
+            return context, sources
+            
+        except Exception as e:
+            logger.warning(f"Vector search failed: {e}")
+            return "", []
+
+    def graph_search(self, question: str, limit: int = 5):
+        """Perform graph-based search using relationships"""
+        try:
+            # Search for connected information in the graph
             search_query = """
+            // Find nodes matching the question
             MATCH (n)
             WHERE n.content IS NOT NULL 
             AND (toLower(n.content) CONTAINS toLower($question) 
                 OR toLower(n.text) CONTAINS toLower($question)
                 OR toLower(n.title) CONTAINS toLower($question))
-            RETURN n.content as content, n.text as text, n.title as title, 
-                   n.source as source, labels(n) as labels
-            LIMIT 5
+            
+            // Get connected nodes for additional context
+            OPTIONAL MATCH (n)-[r]-(connected)
+            WHERE connected.content IS NOT NULL OR connected.text IS NOT NULL
+            
+            WITH n, collect(DISTINCT connected) as connected_nodes
+            
+            RETURN n.content as content, n.text as text, n.title as title,
+                   n.source as source, labels(n) as labels,
+                   [node IN connected_nodes | {
+                       content: coalesce(node.content, node.text),
+                       title: node.title,
+                       relationship: 'connected'
+                   }][0..3] as connected_info
+            LIMIT $limit
             """
             
-            results = self.graph.query(search_query, {"question": question})
+            results = self.graph.query(search_query, {"question": question, "limit": limit})
             
             context_parts = []
             sources = []
@@ -252,18 +388,86 @@ class Neo4jRAGSystem:
                 content = result.get("content") or result.get("text") or ""
                 if content:
                     context_parts.append(content)
+                    
+                    # Add connected context if available
+                    connected_info = result.get("connected_info", [])
+                    for conn in connected_info:
+                        if conn and conn.get("content"):
+                            context_parts.append(f"Related: {conn['content']}")
+                    
                     sources.append({
-                        "source": result.get("source", "Neo4j Graph"),
+                        "source": result.get("source", "Graph Search"),
                         "title": result.get("title", "Unknown"),
-                        "labels": result.get("labels", [])
+                        "labels": result.get("labels", []),
+                        "search_type": "graph",
+                        "connected_nodes": len(connected_info)
                     })
             
             return "\n\n".join(context_parts), sources
             
         except Exception as e:
-            logger.error(f"Fallback search failed: {e}")
+            logger.warning(f"Graph search failed: {e}")
             return "", []
 
+    def comprehensive_search(self, question: str, document_names: List[str] = None):
+        """Perform comprehensive search combining vector, graph, and fulltext"""
+        all_context = []
+        all_sources = []
+        search_results = {"vector": 0, "graph": 0, "fulltext": 0}
+        
+        # 1. Vector Search
+        try:
+            vector_context, vector_sources = self.vector_search(question, document_names, k=3)
+            if vector_context:
+                all_context.append(f"=== SEMANTIC SIMILARITY RESULTS ===\n{vector_context}")
+                all_sources.extend(vector_sources)
+                search_results["vector"] = len(vector_sources)
+        except Exception as e:
+            logger.warning(f"Vector search in comprehensive mode failed: {e}")
+        
+        # 2. Graph Search
+        try:
+            graph_context, graph_sources = self.graph_search(question, limit=3)
+            if graph_context:
+                all_context.append(f"=== GRAPH RELATIONSHIP RESULTS ===\n{graph_context}")
+                all_sources.extend(graph_sources)
+                search_results["graph"] = len(graph_sources)
+        except Exception as e:
+            logger.warning(f"Graph search in comprehensive mode failed: {e}")
+        
+        # 3. Fulltext Search
+        try:
+            fulltext_context, fulltext_sources = self.fulltext_search(question, document_names, limit=3)
+            if fulltext_context:
+                all_context.append(f"=== KEYWORD SEARCH RESULTS ===\n{fulltext_context}")
+                all_sources.extend(fulltext_sources)
+                search_results["fulltext"] = len(fulltext_sources)
+        except Exception as e:
+            logger.warning(f"Fulltext search in comprehensive mode failed: {e}")
+        
+        # Combine and deduplicate sources
+        unique_sources = []
+        seen_sources = set()
+        for source in all_sources:
+            source_key = f"{source.get('source', '')}-{source.get('title', '')}"
+            if source_key not in seen_sources:
+                seen_sources.add(source_key)
+                unique_sources.append(source)
+        
+        final_context = "\n\n".join(all_context)
+        
+        # Add search summary to sources
+        search_summary = {
+            "source": "Search Summary",
+            "title": f"Combined Search Results",
+            "search_type": "comprehensive",
+            "results_breakdown": search_results,
+            "total_sources": len(unique_sources)
+        }
+        unique_sources.insert(0, search_summary)
+        
+        return final_context, unique_sources
+
     def create_retriever(self, document_names: List[str], mode_settings: Dict):
         """Create retriever based on mode and documents"""
         if mode_settings["use_vector_search"] and self.vector_store:
@@ -286,53 +490,82 @@ class Neo4jRAGSystem:
 
     def process_chat_response(self, messages: List, history, question: str, 
                             document_names: List[str], chat_mode_settings: Dict, session_id: str):
-        """Process chat response for vector/hybrid modes"""
+        """Process chat response for vector/hybrid/comprehensive modes"""
         start_time = time.time()
         
         try:
             context = ""
             sources = []
-            
-            # Try vector search first if available
-            if chat_mode_settings["use_vector_search"] and self.vector_store:
-                try:
-                    retriever = self.create_retriever(document_names, chat_mode_settings)
-                    if retriever:
-                        relevant_docs = retriever.invoke(question)
-                        context = "\n\n".join([doc.page_content for doc in relevant_docs])
-                        sources = [
-                            {
-                                "source": doc.metadata.get("source", "Unknown"),
-                                "content": doc.page_content[:200] + "..." if len(doc.page_content) > 200 else doc.page_content
-                            }
-                            for doc in relevant_docs
-                        ]
-                except Exception as e:
-                    logger.warning(f"Vector search failed, using fallback: {e}")
-            
-            # Use fallback search if vector search failed or unavailable
+            search_method = "standard"
+            
+            # Choose search strategy based on mode
+            if chat_mode_settings["mode"] == "comprehensive":
+                context, sources = self.comprehensive_search(question, document_names)
+                search_method = "comprehensive"
+            elif chat_mode_settings["use_vector_search"] and chat_mode_settings["use_graph_search"]:
+                # Hybrid mode: combine vector and graph
+                vector_context, vector_sources = self.vector_search(question, document_names, k=3)
+                graph_context, graph_sources = self.graph_search(question, limit=3)
+                
+                context_parts = []
+                if vector_context:
+                    context_parts.append(f"=== VECTOR SEARCH RESULTS ===\n{vector_context}")
+                if graph_context:
+                    context_parts.append(f"=== GRAPH SEARCH RESULTS ===\n{graph_context}")
+                
+                context = "\n\n".join(context_parts)
+                sources = vector_sources + graph_sources
+                search_method = "hybrid"
+                
+            elif chat_mode_settings["use_vector_search"]:
+                context, sources = self.vector_search(question, document_names)
+                search_method = "vector"
+            elif chat_mode_settings["use_fulltext_search"]:
+                context, sources = self.fulltext_search(question, document_names)
+                search_method = "fulltext"
+            elif chat_mode_settings["use_graph_search"]:
+                context, sources = self.graph_search(question)
+                search_method = "graph"
+            
+            # Fallback if no context found
             if not context:
                 context, fallback_sources = self.fallback_search(question, document_names)
                 sources.extend(fallback_sources)
+                search_method += "_with_fallback"
             
-            # Create prompt template
-            if context:
-                prompt_template = ChatPromptTemplate.from_messages([
-                    ("system", """You are a helpful AI assistant. Use the following context to answer the user's question.
-                    
+            # Create enhanced prompt template for comprehensive mode
+            if chat_mode_settings["mode"] == "comprehensive":
+                system_message = """You are a highly capable AI assistant with access to comprehensive search results from multiple sources:
+
+📊 **SEARCH METHODS USED:**
+- 🔍 **Vector Search**: Semantic similarity using embeddings
+- 🕸️ **Graph Search**: Relationship-based knowledge traversal  
+- 📝 **Fulltext Search**: Keyword and phrase matching
+
+The context below contains results from all these search methods. Use this comprehensive information to provide the most accurate and complete answer possible.
+
 Context:
 {context}
 
-If you cannot find the answer in the context, say so clearly. Always be accurate and helpful."""),
-                    MessagesPlaceholder(variable_name="chat_history"),
-                    ("human", "{question}")
-                ])
+**Instructions:**
+- Synthesize information from all search methods
+- Prioritize accuracy and completeness
+- Mention when information comes from relationships vs. direct content
+- If conflicting information exists, note the discrepancies
+- Cite sources when possible"""
             else:
-                prompt_template = ChatPromptTemplate.from_messages([
-                    ("system", "You are a helpful AI assistant. Answer the user's question to the best of your ability."),
-                    MessagesPlaceholder(variable_name="chat_history"),
-                    ("human", "{question}")
-                ])
+                system_message = """You are a helpful AI assistant. Use the following context to answer the user's question.
+                
+Context:
+{context}
+
+If you cannot find the answer in the context, say so clearly. Always be accurate and helpful."""
+            
+            prompt_template = ChatPromptTemplate.from_messages([
+                ("system", system_message),
+                MessagesPlaceholder(variable_name="chat_history"),
+                ("human", "{question}")
+            ])
             
             # Format chat history (exclude current question)
             chat_history = messages[:-1]
@@ -347,7 +580,14 @@ If you cannot find the answer in the context, say so clearly. Always be accurate
                     "question": question
                 })
             else:
-                response = chain.invoke({
+                # No context found, use LLM knowledge
+                no_context_prompt = ChatPromptTemplate.from_messages([
+                    ("system", "You are a helpful AI assistant. The user is asking about information that may not be in the knowledge base. Answer based on your general knowledge while noting that you don't have specific context from their documents."),
+                    MessagesPlaceholder(variable_name="chat_history"),
+                    ("human", "{question}")
+                ])
+                no_context_chain = no_context_prompt | self.llm | StrOutputParser()
+                response = no_context_chain.invoke({
                     "chat_history": chat_history,
                     "question": question
                 })
@@ -368,6 +608,7 @@ If you cannot find the answer in the context, say so clearly. Always be accurate
                     "total_tokens": 0,
                     "response_time": response_time,
                     "mode": chat_mode_settings["mode"],
+                    "search_method": search_method,
                     "entities": [],
                     "metric_details": [],
                 },
@@ -386,12 +627,50 @@ If you cannot find the answer in the context, say so clearly. Always be accurate
                     "total_tokens": 0,
                     "response_time": 0,
                     "mode": chat_mode_settings["mode"],
+                    "search_method": "error",
                     "entities": [],
                     "metric_details": [],
                 },
                 "user": "chatbot"
             }
 
+    def fallback_search(self, question: str, document_names: List[str] = None):
+        """Fallback search using direct Neo4j queries"""
+        try:
+            # Search for relevant nodes in the graph
+            search_query = """
+            MATCH (n)
+            WHERE n.content IS NOT NULL 
+            AND (toLower(n.content) CONTAINS toLower($question) 
+                OR toLower(n.text) CONTAINS toLower($question)
+                OR toLower(n.title) CONTAINS toLower($question))
+            RETURN n.content as content, n.text as text, n.title as title, 
+                   n.source as source, labels(n) as labels
+            LIMIT 5
+            """
+            
+            results = self.graph.query(search_query, {"question": question})
+            
+            context_parts = []
+            sources = []
+            
+            for result in results:
+                content = result.get("content") or result.get("text") or ""
+                if content:
+                    context_parts.append(content)
+                    sources.append({
+                        "source": result.get("source", "Neo4j Graph"),
+                        "title": result.get("title", "Unknown"),
+                        "labels": result.get("labels", []),
+                        "search_type": "fallback"
+                    })
+            
+            return "\n\n".join(context_parts), sources
+            
+        except Exception as e:
+            logger.error(f"Fallback search failed: {e}")
+            return "", []
+
     def QA_RAG(self, question: str, document_names: str = "[]", 
                session_id: str = None, mode: str = CHAT_GRAPH_MODE, write_access: bool = True):
         """Main QA RAG function"""
@@ -521,13 +800,22 @@ def create_gradio_interface():
 - Session ID: {result['session_id']}"""
             
             if result['info']['sources']:
-                info_text += "\n\n**Sources:**"
+                info_text += "\n\n**Sources & Search Methods:**"
                 for i, source in enumerate(result['info']['sources'], 1):
                     source_name = source.get('source', 'Unknown')
                     source_title = source.get('title', '')
-                    info_text += f"\n{i}. {source_name}"
-                    if source_title:
+                    search_type = source.get('search_type', 'unknown')
+                    
+                    info_text += f"\n{i}. **{source_name}**"
+                    if source_title and source_title != 'Unknown':
                         info_text += f" - {source_title}"
+                    if search_type != 'unknown':
+                        info_text += f" `({search_type})`"
+                    
+                    # Add special info for comprehensive search summary
+                    if source.get('results_breakdown'):
+                        breakdown = source['results_breakdown']
+                        info_text += f"\n   📊 Results: Vector({breakdown['vector']}), Graph({breakdown['graph']}), Fulltext({breakdown['fulltext']})"
             
             status = "✅ Query completed successfully"
             
@@ -568,10 +856,16 @@ def create_gradio_interface():
                 
                 with gr.Row():
                     mode_dropdown = gr.Dropdown(
-                        choices=[CHAT_GRAPH_MODE, CHAT_VECTOR_MODE, CHAT_FULLTEXT_MODE, CHAT_HYBRID_MODE],
-                        value=CHAT_GRAPH_MODE,
-                        label="Chat Mode",
-                        info="Graph mode is most reliable for this setup"
+                        choices=[
+                            CHAT_COMPREHENSIVE_MODE, 
+                            CHAT_GRAPH_MODE, 
+                            CHAT_VECTOR_MODE, 
+                            CHAT_HYBRID_MODE,
+                            CHAT_FULLTEXT_MODE
+                        ],
+                        value=CHAT_COMPREHENSIVE_MODE,
+                        label="Search Mode",
+                        info="Comprehensive mode uses Graph+Vector+Fulltext for best results"
                     )
                     
                     session_input = gr.Textbox(
@@ -624,13 +918,26 @@ def create_gradio_interface():
         # Add examples
         gr.Examples(
             examples=[
-                ["What information do you have about machine learning?", "[]", "graph"],
-                ["Tell me about the documents in the database", "[]", "graph"],
+                ["What information do you have about machine learning?", "[]", "comprehensive"],
+                ["Tell me about the documents in the database", "[]", "graph"], 
+                ["Search for specific keywords in the content", "[]", "fulltext"],
                 ["What are the main topics covered?", "[]", "vector"],
+                ["Find connections between different concepts", "[]", "hybrid"],
             ],
             inputs=[question_input, document_input, mode_dropdown],
         )
-    
+        
+        # Add mode descriptions
+        gr.Markdown("""
+        ## 🔍 **Search Mode Descriptions:**
+        
+        - **🎯 Comprehensive**: Uses Graph+Vector+Fulltext search for the most complete results
+        - **🕸️ Graph**: Explores relationships and connections in your knowledge graph
+        - **🧠 Vector**: Semantic similarity search using AI embeddings  
+        - **🔗 Hybrid**: Combines vector similarity with graph relationships
+        - **📝 Fulltext**: Traditional keyword and phrase search
+        """)
+        
     return demo
 
 def main():
@@ -670,4 +977,18 @@ def QA_RAG_standalone(question: str, document_names: str = "[]",
         }
 
 if __name__ == "__main__":
-    main()
+    main()
+
+# For Hugging Face Spaces deployment, you might also want to create requirements.txt:
+"""
+# requirements.txt content:
+langchain==0.1.0
+langchain-neo4j==0.0.5
+langchain-openai==0.0.8
+langchain-community==0.0.13
+gradio==4.15.0
+python-dotenv==1.0.0
+neo4j==5.16.0
+openai==1.10.0
+tiktoken==0.5.2
+"""