trying fixes for loading

src/proxy_lite/browser/browser.py

@@ -179,116 +179,60 @@ class BrowserSession:
     # re-run for cases of mid-run redirects
     @retry(
         wait=wait_exponential(multiplier=1, min=1, max=10),
-        stop=stop_after_delay(5),
+        stop=stop_after_delay(5),  # This retry is for the entire update_poi method, in case of intermittent issues
         reraise=True,
         before_sleep=before_sleep_log(logger, logging.ERROR),
     )
     async def update_poi(self) -> None:
         try:
-            # Added for robustness based on previous discussions
-            await self.current_page.wait_for_load_state("networkidle", timeout=180000)
+            # Step 1: Wait for network to be idle. This indicates that initial requests have settled.
+            logger.debug("Attempting wait_for_load_state('networkidle')...")
+            await self.current_page.wait_for_load_state("networkidle", timeout=180000) # Increased timeout
             logger.debug("wait_for_load_state('networkidle') completed.")
 
-            # This is the line that was previously timing out, now with increased timeout.
-            # Adding explicit try/except around it for specific debugging.
+            # Step 2: Wait for the 'loading' class to disappear from the body.
+            # This is a common and effective way to detect when SPAs like Salesforce are visually ready.
+            logger.debug("Attempting wait_for_selector('body:not(.loading)')...")
+            # Removed state="visible" as it's often too strict for 'body' in SPAs,
+            # and 'not(.loading)' implies it should become visible eventually.
+            await self.current_page.wait_for_selector("body:not(.loading)", timeout=180000)
+            logger.debug("wait_for_selector('body:not(.loading)') completed.")
+
+            # Optional Step 3 (Highly Recommended): If the above still times out,
+            # uncomment and replace with a reliable selector for an interactive element
+            # that only appears after the Salesforce UI is fully loaded and ready for user input.
+            # Example: await self.current_page.wait_for_selector("#some_salesforce_specific_id", timeout=180000, state="visible")
+            # Example: await self.current_page.wait_for_selector("text=App Launcher", timeout=180000, state="visible")
+            # For now, we'll rely on the 'body:not(.loading)' as the primary indicator.
+
+        except PlaywrightTimeoutError as e:
+            # --- START TEMPORARY DEBUGGING CODE ---
+            # This block captures state specifically when a Playwright timeout occurs
+            current_url = self.current_page.url if self.current_page else "N/A"
+            logger.error(f"DEBUGGING: Playwright Timeout (180s) during page readiness check at URL: {current_url}")
+
+            html_content = None
             try:
-                await self.current_page.wait_for_selector("body", timeout=180000, state="visible")
-                logger.debug("wait_for_selector('body', state='visible') completed.")
-            except PlaywrightTimeoutError as e:
-                # --- START TEMPORARY DEBUGGING CODE ---
-                current_url = self.current_page.url if self.current_page else "N/A"
-                logger.error(f"DEBUGGING: Playwright Timeout (180s) on body selector at URL: {current_url}")
-
-                html_content = None
-                try:
-                    if self.current_page:
-                        html_content = await self.current_page.content()
-                        # Log only a snippet of HTML to avoid excessively large logs
-                        logger.error(f"DEBUGGING: HTML Content (first 1000 chars) when timeout occurred:\n{html_content[:1000]}...")
-                except Exception as html_e:
-                    logger.error(f"DEBUGGING: Could not get HTML content: {html_e}")
-
-                screenshot_b64 = "N/A"
-                try:
-                    if self.current_page:
-                        # Capture screenshot at lower quality to keep log size manageable
-                        screenshot_bytes = await self.current_page.screenshot(type="jpeg", quality=50)
-                        screenshot_b64 = base64.b64encode(screenshot_bytes).decode("utf-8")
-                        # Log only a very short snippet of base64 string
-                        logger.error(f"DEBUGGING: Base64 Screenshot (truncated) when timeout occurred:\ndata:image/jpeg;base64,{screenshot_b64[:100]}... (full string is much longer)")
-                except Exception as ss_e:
-                    logger.error(f"DEBUGGING: Could not take screenshot: {ss_e}")
-
-                # Re-raise the original exception to ensure the task still fails,
-                # but now with crucial debugging information in the logs.
-                raise e
-                # --- END TEMPORARY DEBUGGING CODE ---
-
-        except PlaywrightTimeoutError: # This outer catch is for the wait_for_load_state timeout
-            logger.error(f"Timeout waiting for website load state (networkidle): {self.current_url}")
-            raise # Re-raise if load_state itself times out
-
-        except Exception as e:
-            logger.error(f"An unexpected error occurred during page readiness check: {e}")
-            raise
-
-        # Run the bounding box javascript code to highlight the points of interest on the page
-        page_info = await self.current_page.evaluate(
-            """() => {
-                overwriteDefaultSelectConvergence();
-                return findPOIsConvergence();
-            }""",
-        )
-        # Get the points of interest on the page
-        self.poi_elements = page_info["element_descriptions"]
-        element_centroids = page_info["element_centroids"]
-        try:
-            # Select all iframes on the page
-            iframes = await self.current_page.query_selector_all("iframe")
-
-            max_iframes = 10
-
-            # Define an asynchronous function to process and filter each iframe
-            tasks = [asyncio.create_task(self.process_iframe(iframe)) for iframe in iframes[:max_iframes]]
+                if self.current_page:
+                    html_content = await self.current_page.content()
+                    logger.error(f"DEBUGGING: HTML Content (first 1000 chars) when timeout occurred:\n{html_content[:1000]}...")
+            except Exception as html_e:
+                logger.error(f"DEBUGGING: Could not get HTML content for debug: {html_e}")
 
-            results = await asyncio.gather(*tasks)
-
-            filtered_results = [result for result in results if result is not None]
-
-            iframes_pois = []
-            iframe_offsets = []
-
-            for poi, offset in filtered_results:
-                iframes_pois.append(poi)
-                iframe_offsets.append(offset)
-
-            # Combine the points of interest from the iframes with the main page and adjust the centroids
-            for index, iframe_poi in enumerate(iframes_pois):
-                self.poi_elements.extend(iframe_poi["element_descriptions"])
-                for centroid in iframe_poi["element_centroids"]:
-                    centroid["x"] += iframe_offsets[index]["x"]
-                    centroid["y"] += iframe_offsets[index]["y"]
-                    centroid["left"] += iframe_offsets[index]["x"]
-                    centroid["top"] += iframe_offsets[index]["y"]
-                    centroid["right"] += iframe_offsets[index]["x"]
-                    centroid["bottom"] += iframe_offsets[index]["y"]
-                element_centroids.extend(iframe_poi["element_centroids"])
-
-        except Exception as e:
-            logger.error(f"Error in finding iframes: {e}")
-
-        # Get the centroids of the points of interest
-        self.poi_centroids = [Point(x=xy["x"], y=xy["y"]) for xy in element_centroids]
-        self.bounding_boxes = [BoundingBox(**xy, label=str(i)) for i, xy in enumerate(element_centroids)]
-        self.pois = [
-            POI(info=info, element_centroid=centroid, bounding_box=bbox)
-            for info, centroid, bbox in zip(
-                self.poi_elements,
-                self.poi_centroids,
-                self.bounding_boxes,
-                strict=False,
-            )
-        ]
+            screenshot_b64 = "N/A"
+            try:
+                if self.current_page:
+                    # Capture screenshot at lower quality (e.g., 50) to keep log size manageable.
+                    # Higher quality might make logs too large for some platforms.
+                    screenshot_bytes = await self.current_page.screenshot(type="jpeg", quality=50)
+                    screenshot_b64 = base64.b64encode(screenshot_bytes).decode("utf-8")
+                    # Log only a very short snippet of base64 string to confirm it's there
+                    logger.error(f"DEBUGGING: Base64 Screenshot (truncated) when timeout occurred:\ndata:image/jpeg;base64,{screenshot_b64[:100]}... (full string is much longer)")
+                
+                # If you want to view the full screenshot locally during development, you can save it:
+                # with open("debug_timeout_full_screenshot.jpeg", "wb") as f:
+                #     f.write(screenshot_bytes)
+                # logger.error("DEBUGGING: Full screenshot saved to debug_timeout
 
     @property
     def poi_text(self) -> str: