Update app.py

app.py

@@ -5,17 +5,21 @@ from urllib.parse import urljoin, urlparse
 from fpdf import FPDF
 import tempfile
 import re
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
 
 def clean_text(text):
-    # Remove any non-printable characters
     text = ''.join(char for char in text if char.isprintable())
-    # Replace any remaining problematic characters
     text = re.sub(r'[^\x00-\x7F]+', ' ', text)
     return text
 
 def get_page_content(url):
     try:
-        response = requests.get(url)
+        logger.info(f"Fetching content from: {url}")
+        response = requests.get(url, timeout=10)
+        response.raise_for_status()
         soup = BeautifulSoup(response.text, 'html.parser')
         content = []
         for tag in ['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li']:
@@ -23,8 +27,10 @@ def get_page_content(url):
                 text = clean_text(element.get_text(strip=True))
                 if text:
                     content.append(text)
+        logger.info(f"Found {len(content)} content items for {url}")
         return content
     except Exception as e:
+        logger.error(f"Error processing {url}: {str(e)}")
         return [f"Error processing {url}: {str(e)}"]
 
 def get_subdirectory_pages(url, base_url, visited=set(), max_pages=100):
@@ -35,7 +41,8 @@ def get_subdirectory_pages(url, base_url, visited=set(), max_pages=100):
     pages = [(url, get_page_content(url))]
     
     try:
-        response = requests.get(url)
+        response = requests.get(url, timeout=10)
+        response.raise_for_status()
         soup = BeautifulSoup(response.text, 'html.parser')
         
         for link in soup.find_all('a'):
@@ -47,14 +54,16 @@ def get_subdirectory_pages(url, base_url, visited=set(), max_pages=100):
                     if len(visited) >= max_pages:
                         break
     except Exception as e:
-        print(f"Error processing {url}: {e}")
+        logger.error(f"Error processing {url}: {e}")
     
     return pages
 
 def website_to_pdf(url):
     parsed_url = urlparse(url)
     base_url = f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path.rstrip('/')}/"
+    logger.info(f"Starting to process: {base_url}")
     all_pages = get_subdirectory_pages(base_url, base_url)
+    logger.info(f"Found {len(all_pages)} pages to process")
     
     pdf = FPDF()
     pdf.add_page()
@@ -63,20 +72,22 @@ def website_to_pdf(url):
     for page_url, content in all_pages:
         pdf.cell(200, 10, txt=page_url, ln=True)
         for text in content:
-            pdf.multi_cell(0, 10, txt=text)
+            pdf.multi_cell(0, 10, txt=text[:200])  # Limit text length to avoid issues
         pdf.add_page()
 
     with tempfile.NamedTemporaryFile(delete=False, suffix='.pdf') as tmp:
-        pdf.output(tmp.name)
-        output_file = tmp.name
+        pdf_path = tmp.name
+        pdf.output(pdf_path)
+        logger.info(f"PDF saved to: {pdf_path}")
     
-    return output_file
+    return pdf_path
 
 def process_url(url):
     try:
         pdf_file = website_to_pdf(url)
         return pdf_file
     except Exception as e:
+        logger.error(f"Error in process_url: {str(e)}")
         return f"An error occurred: {str(e)}"
 
 iface = gr.Interface(