Update app.py

app.py

@@ -49,10 +49,19 @@ def get_subdirectory_pages(url, base_url, visited=set(), max_pages=100):
             href = link.get('href')
             if href:
                 full_url = urljoin(base_url, href)
-                if full_url.startswith(base_url) and full_url not in visited:
-                    pages.extend(get_subdirectory_pages(full_url, base_url, visited, max_pages))
-                    if len(visited) >= max_pages:
-                        break
+                parsed_full_url = urlparse(full_url)
+                parsed_base_url = urlparse(base_url)
+                
+                # Check if the URL is one level deeper
+                if (parsed_full_url.scheme == parsed_base_url.scheme and
+                    parsed_full_url.netloc == parsed_base_url.netloc and
+                    parsed_full_url.path.startswith(parsed_base_url.path) and
+                    parsed_full_url.path.count('/') == parsed_base_url.path.count('/') + 1):
+                    
+                    if full_url not in visited:
+                        pages.extend(get_subdirectory_pages(full_url, full_url, visited, max_pages))
+                        if len(visited) >= max_pages:
+                            break
     except Exception as e:
         logger.error(f"Error processing {url}: {e}")