Update app.py

app.py

@@ -1,16 +1,9 @@
 import gradio as gr
-import requests
-from bs4 import BeautifulSoup
-from requests.adapters import HTTPAdapter
-from requests.packages.urllib3.util.retry import Retry
+from requests_html import HTMLSession
 import re
-from selenium import webdriver
-import os
 
 def setup_session():
-    session = requests.Session()
-    retries = Retry(total=5, backoff_factor=1, status_forcelist=[502, 503, 504])
-    session.mount('https://', HTTPAdapter(max_retries=retries))
+    session = HTMLSession()
     return session
 
 def generate_naver_search_url(query):
@@ -23,15 +16,15 @@ def generate_naver_search_url(query):
 def crawl_naver_search_results(url):
     session = setup_session()
     response = session.get(url)
-    soup = BeautifulSoup(response.text, "html.parser")
+    response.html.render()
     results = []
     i = 1
-    for li in soup.find_all("li", class_=re.compile("bx.*")):
-        for div in li.find_all("div", class_="detail_box"):
-            for div2 in div.find_all("div", class_="title_area"):
+    for li in response.html.find("li.bx"):
+        for div in li.find("div.detail_box"):
+            for div2 in div.find("div.title_area"):
                 title = div2.text.strip()
-                for a in div2.find_all("a", href=True):
-                    link = a["href"]
+                for a in div2.find("a", href=True):
+                    link = a.attrs["href"]
                     results.append({"번호": i, "제목": title, "링크": link})
                     i += 1
     html_table = "<table><tr><th>번호</th><th>제목</th><th>링크</th></tr>"
@@ -41,15 +34,12 @@ def crawl_naver_search_results(url):
     return html_table
 
 def get_blog_content(link):
-    options = webdriver.ChromeOptions()
-    options.add_argument("--disable-javascript")
-    os.environ["CHROMEDRIVER_PATH"] = "/usr/local/bin/chromedriver"
-    driver = webdriver.Chrome(options=options)
-    driver.get(link)
-    soup = BeautifulSoup(driver.page_source, "html.parser")
+    session = setup_session()
+    response = session.get(link)
+    response.html.render()
     title = ""
-    for component in soup.find_all("div", class_="se-component se-text se-l-default"):
-        for paragraph in component.find_all("p", class_="se-text-paragraph"):
+    for component in response.html.find("div.se-component.se-text.se-l-default"):
+        for paragraph in component.find("p.se-text-paragraph"):
             title += paragraph.text.strip() + "\n"
     return title