Update app.py

app.py

@@ -2,11 +2,16 @@ import gradio as gr
 import requests
 from bs4 import BeautifulSoup
 from urllib.parse import urljoin, urlparse
-from reportlab.lib.pagesizes import letter
-from reportlab.lib.styles import getSampleStyleSheet
-from reportlab.platypus import SimpleDocTemplate, Paragraph
-from io import BytesIO
+from fpdf import FPDF
 import tempfile
+import re
+
+def clean_text(text):
+    # Remove any non-printable characters
+    text = ''.join(char for char in text if char.isprintable())
+    # Replace any remaining problematic characters
+    text = re.sub(r'[^\x00-\x7F]+', ' ', text)
+    return text
 
 def get_page_content(url):
     try:
@@ -15,15 +20,15 @@ def get_page_content(url):
         content = []
         for tag in ['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li']:
             for element in soup.find_all(tag):
-                text = element.get_text(strip=True)
+                text = clean_text(element.get_text(strip=True))
                 if text:
                     content.append(text)
         return content
     except Exception as e:
         return [f"Error processing {url}: {str(e)}"]
 
-def get_subdirectory_pages(url, base_url, visited=set()):
-    if url in visited:
+def get_subdirectory_pages(url, base_url, visited=set(), max_pages=100):
+    if url in visited or len(visited) >= max_pages:
         return []
     
     visited.add(url)
@@ -38,7 +43,9 @@ def get_subdirectory_pages(url, base_url, visited=set()):
             if href:
                 full_url = urljoin(base_url, href)
                 if full_url.startswith(base_url) and full_url not in visited:
-                    pages.extend(get_subdirectory_pages(full_url, base_url, visited))
+                    pages.extend(get_subdirectory_pages(full_url, base_url, visited, max_pages))
+                    if len(visited) >= max_pages:
+                        break
     except Exception as e:
         print(f"Error processing {url}: {e}")
     
@@ -49,21 +56,18 @@ def website_to_pdf(url):
     base_url = f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path.rstrip('/')}/"
     all_pages = get_subdirectory_pages(base_url, base_url)
     
-    buffer = BytesIO()
-    doc = SimpleDocTemplate(buffer, pagesize=letter)
-    styles = getSampleStyleSheet()
-    story = []
+    pdf = FPDF()
+    pdf.add_page()
+    pdf.set_font("Arial", size=12)
 
     for page_url, content in all_pages:
-        story.append(Paragraph(f"<b>{page_url}</b>", styles['Heading1']))
+        pdf.cell(200, 10, txt=page_url, ln=True)
         for text in content:
-            story.append(Paragraph(text, styles['BodyText']))
-        story.append(Paragraph("<br/><br/>", styles['BodyText']))
+            pdf.multi_cell(0, 10, txt=text)
+        pdf.add_page()
 
-    doc.build(story)
-    
     with tempfile.NamedTemporaryFile(delete=False, suffix='.pdf') as tmp:
-        tmp.write(buffer.getvalue())
+        pdf.output(tmp.name)
         output_file = tmp.name
     
     return output_file