Update middlewares/search_client.py

middlewares/search_client.py

@@ -2,14 +2,15 @@ import requests
 from bs4 import BeautifulSoup
 import re
 import concurrent.futures
+from GoogleNews import GoogleNews
 
 
 class SearchClient:
     def __init__(self, vendor, engine_id=None, api_key=None):
         self.vendor = vendor
-        if vendor == "google":
-            self.endpoint = f"https://www.googleapis.com/customsearch/v1?key={api_key}&cx={engine_id}"
-        elif vendor == "bing":
+        # if vendor == "google":
+        #     self.endpoint = f"https://www.googleapis.com/customsearch/v1?key={api_key}&cx={engine_id}"
+        if vendor == "bing":
             self.endpoint = "https://api.bing.microsoft.com/v7.0/search"
             self.headers = {
                 "Ocp-Apim-Subscription-Key": api_key,
@@ -42,21 +43,25 @@ class SearchClient:
                     print(f"Error fetching data from {link}: {e}")
         return results
 
-    def _google_search(self, query, n_crawl):
-        response = requests.get(self.endpoint, params={"q": query})
-        search_results = response.json()
+    def _google_search(self, start_date,end_date, query, n_crawl):
+        keywords_lst=query.split(',')
+        for keyword in keywords_lst:
+            googlenews = GoogleNews(start=start_date,end=end_date)
+            googlenews = GoogleNews(lang='en', region='IN')
+            googlenews.get_news(keyword)
+            results=googlenews.results
+            texts = googlenews.get_texts()  # List of news texts
+            links = googlenews.get_links()  # List of news links
 
-        results = []
-        count = 0
-        for item in search_results.get("items", []):
-            if count >= n_crawl:
-                break
+            if len(texts)<n_crawl:
+                data = {'Keyword': [keyword]*len(texts), 'Links': links, 'Text': texts}
+            else:
+                data = {'Keyword': [keyword]*(n_crawl), 'Links': links[:n_crawl], 'Text': texts[:n_crawl]}
 
-            link = item["link"]
-            results.append(link)
-            count += 1
+        results=data['Links']
+        corrected_urls = ["https://" + url for url in results]
 
-        text_results = self._fetch_text_from_links(results)
+        text_results = self._fetch_text_from_links(corrected_urls)
         return text_results
 
     def _bing_search(self, query, n_crawl):
@@ -77,9 +82,13 @@ class SearchClient:
         return text_results
 
     def search(self, query, n_crawl):
-        if self.vendor == "google":
-            return self._google_search(query, n_crawl)
-        elif self.vendor == "bing":
+        if self.vendor == "bing":
             return self._bing_search(query, n_crawl)
         else:
             return "Invalid vendor"
+
+    def search_google(self,start_date,end_date, query, n_crawl):
+        if self.vendor == "google":
+            return self._google_search(start_date,end_date,query, n_crawl)
+        else:
+            return "Invalid vendor"