bluenevus commited on
Commit
1b50e52
·
verified ·
1 Parent(s): 56b8e47

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +10 -1
app.py CHANGED
@@ -18,6 +18,10 @@ from contextlib import contextmanager
18
  from threading import local
19
  import time
20
  import os
 
 
 
 
21
 
22
  # Initialize Dash app
23
  app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
@@ -33,6 +37,11 @@ thread_local = local()
33
  # Rate limiter: 10 requests per second
34
  rate_limiter = AsyncLimiter(10, 1)
35
 
 
 
 
 
 
36
  @contextmanager
37
  def get_db_connection():
38
  if not hasattr(thread_local, "connection"):
@@ -107,7 +116,7 @@ async def crawl_pages(base_url, max_depth):
107
  to_visit = [(base_url, 0)]
108
  all_pages = []
109
 
110
- async with aiohttp.ClientSession() as session:
111
  while to_visit:
112
  current_url, depth = to_visit.pop(0)
113
  if current_url in visited or depth > max_depth:
 
18
  from threading import local
19
  import time
20
  import os
21
+ # Create an SSL context that ignores certificate verification
22
+ ssl_context = ssl.create_default_context()
23
+ ssl_context.check_hostname = False
24
+ ssl_context.verify_mode = ssl.CERT_NONE
25
 
26
  # Initialize Dash app
27
  app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
 
37
  # Rate limiter: 10 requests per second
38
  rate_limiter = AsyncLimiter(10, 1)
39
 
40
+ # Create an SSL context that ignores certificate verification
41
+ ssl_context = ssl.create_default_context()
42
+ ssl_context.check_hostname = False
43
+ ssl_context.verify_mode = ssl.CERT_NONE
44
+
45
  @contextmanager
46
  def get_db_connection():
47
  if not hasattr(thread_local, "connection"):
 
116
  to_visit = [(base_url, 0)]
117
  all_pages = []
118
 
119
+ async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=ssl_context)) as session:
120
  while to_visit:
121
  current_url, depth = to_visit.pop(0)
122
  if current_url in visited or depth > max_depth: