Update app.py

app.py

@@ -122,28 +122,32 @@ async def crawl_pages(base_url, max_depth):
             visited.add(current_url)
             start_time = time.time()
 
-            with get_db_connection() as conn:
-                c = conn.cursor()
-                c.execute("SELECT content FROM pages WHERE url = ?", (current_url,))
-                result = c.fetchone()
-
-            if result:
-                content = eval(result[0])  # Convert string back to list
-            else:
-                content = await get_page_content(session, current_url)
+            try:
                 with get_db_connection() as conn:
                     c = conn.cursor()
-                    c.execute("INSERT INTO pages VALUES (?, ?, ?)", (current_url, str(content), depth))
-                    conn.commit()
+                    c.execute("SELECT content FROM pages WHERE url = ?", (current_url,))
+                    result = c.fetchone()
+
+                if result:
+                    content = eval(result[0])  # Convert string back to list
+                else:
+                    content = await get_page_content(session, current_url)
+                    with get_db_connection() as conn:
+                        c = conn.cursor()
+                        c.execute("INSERT INTO pages VALUES (?, ?, ?)", (current_url, str(content), depth))
+                        conn.commit()
 
-            all_pages.append((current_url, content))
-            logger.info(f"Processed page: {current_url} at depth {depth} in {time.time() - start_time:.2f} seconds")
+                all_pages.append((current_url, content))
+                logger.info(f"Processed page: {current_url} at depth {depth} in {time.time() - start_time:.2f} seconds")
 
-            if depth < max_depth:
-                links = await get_links(session, current_url, base_url)
-                for link in links:
-                    if link not in visited:
-                        to_visit.append((link, depth + 1))
+                if depth < max_depth:
+                    links = await get_links(session, current_url, base_url)
+                    for link in links:
+                        if link not in visited:
+                            to_visit.append((link, depth + 1))
+            except Exception as e:
+                logger.error(f"Error processing {current_url}: {str(e)}")
+                # Continue with the next URL even if this one fails
 
     return all_pages
 
@@ -176,6 +180,8 @@ def website_to_pdf(all_pages):
 async def process_url(url, depth):
     try:
         all_pages = await crawl_pages(url, depth)
+        if not all_pages:
+            return "No pages were successfully crawled. Please check the URL and try again."
         pdf_file = website_to_pdf(all_pages)
         return pdf_file
     except Exception as e:
@@ -228,22 +234,28 @@ def update_output(n_clicks, url, depth):
     
     pdf_path = asyncio.run(process_url(url, depth))
     
-    if pdf_path.startswith("An error occurred"):
-        return pdf_path
-    
-    with open(pdf_path, "rb") as f:
-        encoded = base64.b64encode(f.read()).decode()
+    if isinstance(pdf_path, str):
+        return pdf_path  # This is an error message
     
-    os.unlink(pdf_path)  # Remove the temporary file
-    
-    return html.Div([
-        html.H4("PDF Generated Successfully"),
-        html.A(
-            dbc.Button("Download PDF", color="success", className="mt-2"),
-            href=f"data:application/pdf;base64,{encoded}",
-            download="website_content.pdf"
-        )
-    ])
+    try:
+        with open(pdf_path, "rb") as f:
+            encoded = base64.b64encode(f.read()).decode()
+        
+        os.unlink(pdf_path)  # Remove the temporary file
+        
+        return html.Div([
+            html.H4("PDF Generated Successfully"),
+            html.A(
+                dbc.Button("Download PDF", color="success", className="mt-2"),
+                href=f"data:application/pdf;base64,{encoded}",
+                download="website_content.pdf"
+            )
+        ])
+    except Exception as e:
+        logger.error(f"Error creating download link: {str(e)}")
+        return f"An error occurred while creating the download link: {str(e)}"
 
 if __name__ == '__main__':
-    app.run(debug=True, host='0.0.0.0', port=7860)
+    print("Starting the Dash application...")
+    app.run(debug=True, host='0.0.0.0', port=7860)
+    print("Dash application has finished running.")