Update app.py

app.py

@@ -92,7 +92,7 @@ async def get_links(session, url, base_url):
         logger.error(f"Error getting links from {url}: {str(e)}")
         return []
 
-async def crawl_pages(base_url, max_depth):
+ async def crawl_pages(base_url, max_depth):
     visited = set()
     to_visit = [(base_url, 0)]
     all_pages = []
@@ -126,7 +126,7 @@ async def crawl_pages(base_url, max_depth):
             if depth < max_depth:
                 links = await get_links(session, current_url, base_url)
                 for link in links:
-                    if link not in visited:
+                    if link not in visited and link not in [url for url, _ in to_visit]:
                         to_visit.append((link, depth + 1))
 
     return all_pages