Update helpers.py

helpers.py

@@ -21,37 +21,69 @@ if "GOOGLE_API_KEY" not in os.environ:
     os.environ["GOOGLE_API_KEY"] = "AIzaSyDJ4vIKuIBIPNHATLxnoHlagXWbsAz-vRs"
 key = "AIzaSyDJ4vIKuIBIPNHATLxnoHlagXWbsAz-vRs"
 
-def get_web_documents(base_url='https://nct.neu.edu.vn/'):
-    """Lấy tất cả URL từ trang web và trích xuất nội dung văn bản"""
-    
-    # Bước 1: Lấy danh sách URL
-    response = requests.get(base_url)
-    if response.status_code != 200:
-        print(f"Không thể truy cập {base_url}")
-        return []
-    
-    soup = BeautifulSoup(response.text, 'html.parser')
-    
-    urls = set()  # Dùng set để tránh trùng lặp
-    for link in soup.find_all('a', href=True):
-        href = link.get('href')
-        if href.startswith(base_url):  # Chỉ lấy URL nội bộ
-            urls.add(href)
-
-    print(f"Tìm thấy {len(urls)} URL hợp lệ.")
-
-    # Bước 2: Tải nội dung từ các URL
+import asyncio
+from urllib.parse import urljoin
+from playwright.async_api import async_playwright
+from langchain_community.document_loaders import AsyncHtmlLoader
+from langchain.text_splitter import Html2TextTransformer
+from tqdm.asyncio import tqdm
+
+async def _fetch_urls(base_url):
+    """Extract all links from a JavaScript-rendered webpage."""
+    async with async_playwright() as p:
+        browser = await p.chromium.launch(headless=True)
+        page = await browser.new_page()
+        await page.goto(base_url)
+        await page.wait_for_load_state("networkidle")
+
+        urls = set()
+        links = await page.locator("a").all()
+        for link in links:
+            href = await link.get_attribute("href")
+            if href and "#" not in href:
+                full_url = urljoin(base_url, href)
+                if full_url.startswith(base_url):
+                    urls.add(full_url)
+
+        await browser.close()
+    return list(urls)
+
+async def _fetch_web_content(urls):
+    """Fetch HTML content and convert it to text, with a progress bar."""
     docs = []
+    progress_bar = tqdm(total=len(urls), desc="Scraping Pages", unit="page")
+
     for page_url in urls:
         try:
-            loader = WebBaseLoader(web_paths=[page_url])
-            docs.extend(loader.load())  # Đồng bộ, không cần async
+            loader = AsyncHtmlLoader(page_url)
+            html2text = Html2TextTransformer()
+            html = await loader.aload()
+            doc = html2text.transform_documents(html)
+            docs.extend(doc)
         except Exception as e:
-            print(f"Lỗi khi tải {page_url}: {e}")
+            print(f"Error loading {page_url}: {e}")
 
-    print(f"Tải thành công {len(docs)} trang.")
+        progress_bar.update(1)  # Update progress bar
+
+    progress_bar.close()
     return docs
 
+def scrape_website(base_urls):
+    """
+    Scrapes a list of base URLs and extracts their content.
+    Includes a progress bar for tracking.
+    """
+    async def _main():
+        all_urls = []
+        for base_url in base_urls:
+            urls = await _fetch_urls(base_url)
+            all_urls.extend(urls)
+
+        docs = await _fetch_web_content(all_urls)
+        return docs
+
+    return asyncio.run(_main())
+    
 def log_message(messages, filename="chat_log.txt"):
     """Ghi lịch sử tin nhắn vào file log"""
     with open(filename, "a", encoding="utf-8") as f: