Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
@@ -18,6 +18,10 @@ from contextlib import contextmanager
|
|
18 |
from threading import local
|
19 |
import time
|
20 |
import os
|
|
|
|
|
|
|
|
|
21 |
|
22 |
# Initialize Dash app
|
23 |
app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
|
@@ -33,6 +37,11 @@ thread_local = local()
|
|
33 |
# Rate limiter: 10 requests per second
|
34 |
rate_limiter = AsyncLimiter(10, 1)
|
35 |
|
|
|
|
|
|
|
|
|
|
|
36 |
@contextmanager
|
37 |
def get_db_connection():
|
38 |
if not hasattr(thread_local, "connection"):
|
@@ -107,7 +116,7 @@ async def crawl_pages(base_url, max_depth):
|
|
107 |
to_visit = [(base_url, 0)]
|
108 |
all_pages = []
|
109 |
|
110 |
-
async with aiohttp.ClientSession() as session:
|
111 |
while to_visit:
|
112 |
current_url, depth = to_visit.pop(0)
|
113 |
if current_url in visited or depth > max_depth:
|
|
|
18 |
from threading import local
|
19 |
import time
|
20 |
import os
|
21 |
+
# Create an SSL context that ignores certificate verification
|
22 |
+
ssl_context = ssl.create_default_context()
|
23 |
+
ssl_context.check_hostname = False
|
24 |
+
ssl_context.verify_mode = ssl.CERT_NONE
|
25 |
|
26 |
# Initialize Dash app
|
27 |
app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
|
|
|
37 |
# Rate limiter: 10 requests per second
|
38 |
rate_limiter = AsyncLimiter(10, 1)
|
39 |
|
40 |
+
# Create an SSL context that ignores certificate verification
|
41 |
+
ssl_context = ssl.create_default_context()
|
42 |
+
ssl_context.check_hostname = False
|
43 |
+
ssl_context.verify_mode = ssl.CERT_NONE
|
44 |
+
|
45 |
@contextmanager
|
46 |
def get_db_connection():
|
47 |
if not hasattr(thread_local, "connection"):
|
|
|
116 |
to_visit = [(base_url, 0)]
|
117 |
all_pages = []
|
118 |
|
119 |
+
async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=ssl_context)) as session:
|
120 |
while to_visit:
|
121 |
current_url, depth = to_visit.pop(0)
|
122 |
if current_url in visited or depth > max_depth:
|