Update app.py

app.py

@@ -35,13 +35,18 @@ def get_page_content(url):
         logger.error(f"Error processing {url}: {str(e)}")
         return [f"Error processing {url}: {str(e)}"]
 
-def get_links(url):
+def get_links(url, base_url):
     try:
         response = requests.get(url, timeout=10)
         response.raise_for_status()
         soup = BeautifulSoup(response.text, 'html.parser')
         links = soup.find_all('a', href=True)
-        return [urljoin(url, link['href']) for link in links if link['href'].startswith('/docs')]
+        valid_links = []
+        for link in links:
+            full_url = urljoin(url, link['href'])
+            if full_url.startswith(base_url) and full_url != url:
+                valid_links.append(full_url)
+        return valid_links
     except Exception as e:
         logger.error(f"Error getting links from {url}: {str(e)}")
         return []
@@ -59,10 +64,13 @@ def crawl_pages(base_url, max_depth):
         visited.add(current_url)
         content = get_page_content(current_url)
         all_pages.append((current_url, content))
+        logger.info(f"Processed page: {current_url} at depth {depth}")
 
         if depth < max_depth:
-            links = get_links(current_url)
-            to_visit.extend((link, depth + 1) for link in links if link not in visited)
+            links = get_links(current_url, base_url)
+            for link in links:
+                if link not in visited:
+                    to_visit.append((link, depth + 1))
 
     return all_pages