Update services/faq_service.py

services/faq_service.py

@@ -1,10 +1,11 @@
-# services/faq_service.py
 from typing import List, Dict, Any, Optional
 import aiohttp
 from bs4 import BeautifulSoup
 import faiss
 import logging
 from config.config import settings
+import asyncio
+from urllib.parse import urljoin
 
 logger = logging.getLogger(__name__)
 
@@ -13,54 +14,118 @@ class FAQService:
         self.embedder = model_service.embedder
         self.faiss_index = None
         self.faq_data = []
+        self.visited_urls = set()
+        self.base_url = "https://www.bofrost.de/faq/"
 
     async def fetch_faq_pages(self) -> List[Dict[str, Any]]:
         async with aiohttp.ClientSession() as session:
             try:
-                async with session.get(f"{settings.FAQ_ROOT_URL}sitemap.xml", timeout=settings.TIMEOUT) as response:
-                    if response.status == 200:
-                        sitemap = await response.text()
-                        soup = BeautifulSoup(sitemap, 'xml')
-                        faq_urls = [loc.text for loc in soup.find_all('loc') if "/faq/" in loc.text]
-                        
-                        tasks = [self.fetch_faq_content(url, session) for url in faq_urls]
-                        return await asyncio.gather(*tasks)
+                # Start with the main FAQ page
+                pages = await self.crawl_faq_pages(self.base_url, session)
+                return [page for page in pages if page]
             except Exception as e:
-                logger.error(f"Error fetching FAQ sitemap: {e}")
+                logger.error(f"Error fetching FAQ pages: {e}")
                 return []
 
-    async def fetch_faq_content(self, url: str, session: aiohttp.ClientSession) -> Optional[Dict[str, Any]]:
+    async def crawl_faq_pages(self, url: str, session: aiohttp.ClientSession) -> List[Dict[str, Any]]:
+        if url in self.visited_urls or not url.startswith(self.base_url):
+            return []
+
+        self.visited_urls.add(url)
+        pages = []
+
         try:
             async with session.get(url, timeout=settings.TIMEOUT) as response:
                 if response.status == 200:
                     content = await response.text()
                     soup = BeautifulSoup(content, 'html.parser')
+                    
+                    # Add current page content
+                    page_content = await self.parse_faq_content(soup, url)
+                    if page_content:
+                        pages.append(page_content)
 
-                    faq_title = soup.find('h1').text.strip() if soup.find('h1') else "Unknown Title"
-                    faqs = []
-                    sections = soup.find_all(['div', 'section'], class_=['faq-item', 'faq-section'])
+                    # Find and follow FAQ links
+                    tasks = []
+                    for link in soup.find_all('a', href=True):
+                        href = link['href']
+                        full_url = urljoin(url, href)
+                        
+                        if (full_url.startswith(self.base_url) and 
+                            full_url not in self.visited_urls):
+                            tasks.append(self.crawl_faq_pages(full_url, session))
+                    
+                    if tasks:
+                        results = await asyncio.gather(*tasks)
+                        for result in results:
+                            pages.extend(result)
 
-                    for section in sections:
-                        question = section.find(['h2', 'h3']).text.strip() if section.find(['h2', 'h3']) else None
-                        answer = section.find(['p']).text.strip() if section.find(['p']) else None
+        except Exception as e:
+            logger.error(f"Error crawling FAQ page {url}: {e}")
 
-                        if question and answer:
-                            faqs.append({"question": question, "answer": answer})
+        return pages
 
-                    return {"url": url, "title": faq_title, "faqs": faqs}
+    async def parse_faq_content(self, soup: BeautifulSoup, url: str) -> Optional[Dict[str, Any]]:
+        try:
+            faqs = []
+            faq_items = soup.find_all('div', class_='faq-item')
+            
+            for item in faq_items:
+                # Extract question
+                question_elem = item.find('a', class_='headline-collapse')
+                if not question_elem:
+                    continue
+                    
+                question = question_elem.find('span')
+                if not question:
+                    continue
+                    
+                question_text = question.text.strip()
+                
+                # Extract answer
+                content_elem = item.find('div', class_='content-collapse')
+                if not content_elem:
+                    continue
+                    
+                wysiwyg = content_elem.find('div', class_='wysiwyg-content')
+                if not wysiwyg:
+                    continue
+                
+                # Extract all text while preserving structure
+                answer_parts = []
+                for elem in wysiwyg.find_all(['p', 'li']):
+                    text = elem.get_text(strip=True)
+                    if text:
+                        answer_parts.append(text)
+                
+                answer_text = ' '.join(answer_parts)
+                
+                if question_text and answer_text:
+                    faqs.append({
+                        "question": question_text,
+                        "answer": answer_text
+                    })
+            
+            if faqs:
+                return {
+                    "url": url,
+                    "faqs": faqs
+                }
+            
         except Exception as e:
-            logger.error(f"Error fetching FAQ content from {url}: {e}")
-            return None
+            logger.error(f"Error parsing FAQ content from {url}: {e}")
+        
+        return None
 
     async def index_faqs(self):
         faq_pages = await self.fetch_faq_pages()
-        faq_pages = [page for page in faq_pages if page]
         
         self.faq_data = []
         all_texts = []
-
+        
         for faq_page in faq_pages:
             for item in faq_page['faqs']:
+                # Combine question and answer for better semantic search
                 combined_text = f"{item['question']} {item['answer']}"
                 all_texts.append(combined_text)
                 self.faq_data.append({
@@ -68,7 +133,12 @@ class FAQService:
                     "answer": item['answer'],
                     "source": faq_page['url']
                 })
-
+        
+        if not all_texts:
+            logger.warning("No FAQ content found to index")
+            return
+            
+        # Create embeddings and index them
         embeddings = self.embedder.encode(all_texts, convert_to_tensor=True).cpu().detach().numpy()
         dimension = embeddings.shape[1]
         self.faiss_index = faiss.IndexFlatL2(dimension)
@@ -77,15 +147,19 @@ class FAQService:
     async def search_faqs(self, query: str, top_k: int = 5) -> List[Dict[str, Any]]:
         if not self.faiss_index:
             await self.index_faqs()
-
+            
+        if not self.faq_data:
+            logger.warning("No FAQ data available for search")
+            return []
+            
         query_embedding = self.embedder.encode([query], convert_to_tensor=True).cpu().detach().numpy()
         distances, indices = self.faiss_index.search(query_embedding, top_k)
-
+        
         results = []
         for i, idx in enumerate(indices[0]):
             if idx < len(self.faq_data):
                 result = self.faq_data[idx].copy()
                 result["score"] = float(distances[0][i])
                 results.append(result)
-
+                
         return results