Update app.py

app.py

@@ -22,64 +22,46 @@ def get_page_content(url):
         response.raise_for_status()
         soup = BeautifulSoup(response.text, 'html.parser')
         content = []
-        for tag in ['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li']:
-            for element in soup.find_all(tag):
-                text = clean_text(element.get_text(strip=True))
-                if text:
-                    content.append(text)
+        main_content = soup.find('main')
+        if main_content:
+            for tag in ['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li']:
+                for element in main_content.find_all(tag):
+                    text = clean_text(element.get_text(strip=True))
+                    if text:
+                        content.append(text)
         logger.info(f"Found {len(content)} content items for {url}")
         return content
     except Exception as e:
         logger.error(f"Error processing {url}: {str(e)}")
         return [f"Error processing {url}: {str(e)}"]
 
-def get_subdirectory_pages(url, base_url, visited=set(), max_pages=100):
-    if url in visited or len(visited) >= max_pages:
-        return []
-    
-    visited.add(url)
-    pages = [(url, get_page_content(url))]
-    
+def get_all_doc_links(url):
     try:
         response = requests.get(url, timeout=10)
         response.raise_for_status()
         soup = BeautifulSoup(response.text, 'html.parser')
-        
-        for link in soup.find_all('a'):
-            href = link.get('href')
-            if href:
-                full_url = urljoin(base_url, href)
-                parsed_full_url = urlparse(full_url)
-                parsed_base_url = urlparse(base_url)
-                
-                # Check if the URL is in the same directory or a direct subdirectory
-                if (parsed_full_url.scheme == parsed_base_url.scheme and
-                    parsed_full_url.netloc == parsed_base_url.netloc and
-                    parsed_full_url.path.startswith(parsed_base_url.path) and
-                    parsed_full_url.path.count('/') <= parsed_base_url.path.count('/') + 1):
-                    
-                    if full_url not in visited:
-                        pages.extend(get_subdirectory_pages(full_url, base_url, visited, max_pages))
-                        if len(visited) >= max_pages:
-                            break
+        main_content = soup.find('main')
+        if main_content:
+            links = main_content.find_all('a', href=True)
+            return [urljoin(url, link['href']) for link in links if link['href'].startswith('/docs')]
+        return []
     except Exception as e:
-        logger.error(f"Error processing {url}: {e}")
-    
-    return pages
+        logger.error(f"Error getting links from {url}: {str(e)}")
+        return []
 
 def website_to_pdf(url):
-    parsed_url = urlparse(url)
-    base_url = f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path.rstrip('/')}/"
+    base_url = "https://www.gradio.app/docs"
     logger.info(f"Starting to process: {base_url}")
-    all_pages = get_subdirectory_pages(base_url, base_url)
-    logger.info(f"Found {len(all_pages)} pages to process")
+    all_links = get_all_doc_links(base_url)
+    logger.info(f"Found {len(all_links)} pages to process")
     
     pdf = FPDF()
     pdf.set_auto_page_break(auto=True, margin=15)
     pdf.add_page()
     pdf.set_font("Arial", size=12)
 
-    for page_url, content in all_pages:
+    for page_url in all_links:
+        content = get_page_content(page_url)
         pdf.cell(0, 10, txt=page_url, ln=True)
         pdf.ln(5)
         for text in content:
@@ -103,10 +85,10 @@ def process_url(url):
 
 iface = gr.Interface(
     fn=process_url,
-    inputs=gr.Textbox(label="Enter website URL (e.g., https://www.gradio.app/docs/gradio)"),
+    inputs=gr.Textbox(label="Enter website URL (e.g., https://www.gradio.app/docs)"),
     outputs=gr.File(label="Download PDF"),
-    title="Website Subdirectory to PDF Converter",
-    description="Enter a website URL to convert its subdirectories into a PDF."
+    title="Gradio Documentation to PDF Converter",
+    description="Enter the Gradio docs URL to convert all documentation pages into a PDF."
 )
 
 if __name__ == "__main__":