limitedonly41 commited on
Commit
3734fbf
·
verified ·
1 Parent(s): bc8aa35

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +12 -1
app.py CHANGED
@@ -13,6 +13,8 @@ from curl_cffi.requests import AsyncSession
13
  from tqdm.asyncio import tqdm
14
  from fake_headers import Headers
15
  from urllib.parse import urlparse, urlunparse
 
 
16
 
17
  # Limit the number of concurrent workers
18
  CONCURRENT_WORKERS = 5
@@ -51,6 +53,15 @@ def get_main_page_url(url):
51
  return f"Error processing URL: {e}"
52
 
53
 
 
 
 
 
 
 
 
 
 
54
  async def get_page_bs4(url: str, headers):
55
 
56
  wrong_result = {
@@ -278,7 +289,7 @@ Describe the topic of website from its text :
278
 
279
  summary = tokenizer.decode(outputs[0], skip_special_tokens=True)
280
  final_answer = summary.split("### Response:")[1].strip()
281
- return final_answer
282
 
283
  # Create a Gradio interface
284
  iface = gr.Interface(
 
13
  from tqdm.asyncio import tqdm
14
  from fake_headers import Headers
15
  from urllib.parse import urlparse, urlunparse
16
+ from deep_translator import GoogleTranslator
17
+
18
 
19
  # Limit the number of concurrent workers
20
  CONCURRENT_WORKERS = 5
 
53
  return f"Error processing URL: {e}"
54
 
55
 
56
+ def translate_text(text):
57
+ try:
58
+ text = text[:4990] # Limit the text length to avoid API errors
59
+ translated_text = GoogleTranslator(source='auto', target='en').translate(text)
60
+ return translated_text
61
+ except Exception as e:
62
+ print(f"An error occurred during translation: {e}")
63
+ return None
64
+
65
  async def get_page_bs4(url: str, headers):
66
 
67
  wrong_result = {
 
289
 
290
  summary = tokenizer.decode(outputs[0], skip_special_tokens=True)
291
  final_answer = summary.split("### Response:")[1].strip()
292
+ return f"{main_page_url}: {final_answer}"
293
 
294
  # Create a Gradio interface
295
  iface = gr.Interface(