Update app.py

app.py

@@ -151,7 +151,7 @@ async def crawl_pages(base_url, max_depth):
 
     return all_pages
 
-def website_to_pdf(all_pages):
+def website_to_pdf(all_pages, progress_callback):
     logger.info(f"Starting PDF generation for {len(all_pages)} pages")
     
     pdf = FPDF()
@@ -159,25 +159,31 @@ def website_to_pdf(all_pages):
     pdf.add_page()
     pdf.set_font("Arial", size=12)
 
-    for page_url, content in all_pages:
-        pdf.cell(0, 10, txt=page_url, ln=True)
-        pdf.ln(5)
-        for text in content:
-            try:
-                pdf.multi_cell(0, 10, txt=text[:200])  # Limit text length to avoid issues
-            except Exception as e:
-                logger.error(f"Error writing text to PDF: {str(e)}")
-        if pdf.get_y() > 250:  # Add a new page if the current page is almost full
-            pdf.add_page()
+    batch_size = 100
+    for i in range(0, len(all_pages), batch_size):
+        batch = all_pages[i:i+batch_size]
+        for page_url, content in batch:
+            pdf.cell(0, 10, txt=page_url, ln=True)
+            pdf.ln(5)
+            for text in content:
+                try:
+                    pdf.multi_cell(0, 10, txt=text[:200])  # Limit text length to avoid issues
+                except Exception as e:
+                    logger.error(f"Error writing text to PDF: {str(e)}")
+            if pdf.get_y() > 250:  # Add a new page if the current page is almost full
+                pdf.add_page()
+        
+        progress = min((i + batch_size) / len(all_pages), 1.0)
+        progress_callback(f"Processing pages... {progress:.0%}")
 
     return pdf.output(dest='S').encode('latin-1')  # Return bytes instead of BytesIO object
 
-async def process_url(url, depth):
+async def process_url(url, depth, progress_callback):
     try:
         all_pages = await crawl_pages(url, depth)
         if not all_pages:
             return "No pages were successfully crawled. Please check the URL and try again."
-        pdf_content = website_to_pdf(all_pages)
+        pdf_content = website_to_pdf(all_pages, progress_callback)
         return pdf_content  # This is now bytes, not BytesIO
     except Exception as e:
         logger.error(f"Error in process_url: {str(e)}")
@@ -211,6 +217,7 @@ app.layout = dbc.Container([
             dcc.Slider(id="depth-slider", min=1, max=10, step=1, value=3, marks={i: str(i) for i in range(1, 11)}, className="mb-3"),
             dbc.Button("Convert to PDF", id="submit-button", color="primary", className="mb-3 w-100"),
             dbc.Spinner(html.Div(id="output-area"), color="primary", type="grow"),
+            dcc.Interval(id="progress-interval", interval=1000, n_intervals=0, disabled=True),
         ]),
         className="mt-4"
     )
@@ -218,34 +225,72 @@ app.layout = dbc.Container([
 
 @app.callback(
     Output("output-area", "children"),
+    Output("progress-interval", "disabled"),
     Input("submit-button", "n_clicks"),
+    Input("progress-interval", "n_intervals"),
     State("url-input", "value"),
     State("depth-slider", "value"),
     prevent_initial_call=True
 )
-def update_output(n_clicks, url, depth):
+def update_output(n_clicks, n_intervals, url, depth):
+    ctx = dash.callback_context
+    if not ctx.triggered:
+        raise PreventUpdate
+
+    triggered_id = ctx.triggered[0]['prop_id'].split('.')[0]
+
+    if triggered_id == "submit-button":
+        if not url:
+            return "Please enter a valid URL.", True
+
+        return dcc.Store(id='pdf-store', data='processing'), False
+
+    elif triggered_id == "progress-interval":
+        store = dash.callback_context.inputs.get('pdf-store', None)
+        if store is None or store == 'processing':
+            return "Processing... Please wait.", False
+
+        if isinstance(store, str) and store.startswith("Error"):
+            return store, True
+
+        try:
+            encoded = base64.b64encode(store).decode()
+            return html.Div([
+                html.H4("PDF Generated Successfully"),
+                html.A(
+                    dbc.Button("Download PDF", color="success", className="mt-2"),
+                    href=f"data:application/pdf;base64,{encoded}",
+                    download="website_content.pdf"
+                )
+            ]), True
+        except Exception as e:
+            logger.error(f"Error creating download link: {str(e)}")
+            return f"An error occurred while creating the download link: {str(e)}", True
+
+    raise PreventUpdate
+
+@app.callback(
+    Output('pdf-store', 'data'),
+    Input('submit-button', 'n_clicks'),
+    State('url-input', 'value'),
+    State('depth-slider', 'value'),
+    prevent_initial_call=True
+)
+def generate_pdf(n_clicks, url, depth):
     if not url:
         return "Please enter a valid URL."
-    
-    pdf_content = asyncio.run(process_url(url, depth))
-    
+
+    progress_store = {'progress': 0}
+
+    def progress_callback(message):
+        progress_store['progress'] = message
+
+    pdf_content = asyncio.run(process_url(url, depth, progress_callback))
+
     if isinstance(pdf_content, str):
         return pdf_content  # This is an error message
-    
-    try:
-        encoded = base64.b64encode(pdf_content).decode()
-        
-        return html.Div([
-            html.H4("PDF Generated Successfully"),
-            html.A(
-                dbc.Button("Download PDF", color="success", className="mt-2"),
-                href=f"data:application/pdf;base64,{encoded}",
-                download="website_content.pdf"
-            )
-        ])
-    except Exception as e:
-        logger.error(f"Error creating download link: {str(e)}")
-        return f"An error occurred while creating the download link: {str(e)}"
+
+    return pdf_content
 
 if __name__ == '__main__':
     print("Starting the Dash application...")