Update middlewares/search_client.py

middlewares/search_client.py

@@ -2,7 +2,7 @@ import requests
 from bs4 import BeautifulSoup
 import re
 import concurrent.futures
-from GoogleNews import GoogleNews
+from googlesearch import search
 
 
 class SearchClient:
@@ -43,25 +43,13 @@ class SearchClient:
                     print(f"Error fetching data from {link}: {e}")
         return results
 
-    def _google_search(self, start_date,end_date, query, n_crawl):
-        keywords_lst=query.split(',')
-        for keyword in keywords_lst:
-            googlenews = GoogleNews(start=start_date,end=end_date)
-            googlenews = GoogleNews(lang='en', region='IN')
-            googlenews.get_news(keyword)
-            results=googlenews.results
-            texts = googlenews.get_texts()  # List of news texts
-            links = googlenews.get_links()  # List of news links
+    def _google_search(self,query, n_crawl):
+        search_results = search(query, stop=n_crawl, lang='en', country='IN')
 
-            if len(texts)<n_crawl:
-                data = {'Keyword': [keyword]*len(texts), 'Links': links, 'Text': texts}
-            else:
-                data = {'Keyword': [keyword]*(n_crawl), 'Links': links[:n_crawl], 'Text': texts[:n_crawl]}
+        # Convert the search results to a list
+        links = list(search_results)
 
-        results=data['Links']
-        corrected_urls = ["https://" + url for url in results]
-
-        text_results = self._fetch_text_from_links(corrected_urls)
+        text_results = self._fetch_text_from_links(links)
         return text_results
 
     def _bing_search(self, query, n_crawl):
@@ -87,8 +75,9 @@ class SearchClient:
         else:
             return "Invalid vendor"
 
-    def search_google(self,start_date,end_date, query, n_crawl):
+    def search_google(self, query, n_crawl):
         if self.vendor == "google":
-            return self._google_search(start_date,end_date,query, n_crawl)
+            return self._google_search(query, n_crawl)
         else:
             return "Invalid vendor"
+