Update app.py

app.py

@@ -5,6 +5,11 @@ import os
 from datetime import datetime, timedelta
 from huggingface_hub import InferenceClient
 
+from bs4 import BeautifulSoup
+import concurrent.futures
+import time
+import re
+
 MAX_COUNTRY_RESULTS = 100    # 국가별 최대 결과 수
 MAX_GLOBAL_RESULTS = 1000    # 전세계 최대 결과 수
 
@@ -311,6 +316,9 @@ def serphouse_search(query, country):
     return format_results_from_raw(response_data)
 
 
+
+
+
 # Hacker News API 관련 함수들 먼저 추가
 def get_hn_item(item_id):
     """개별 아이템 정보 가져오기"""
@@ -351,33 +359,134 @@ def format_hn_time(timestamp):
     except:
         return "Unknown time"
 
+
+
+
+
+
+
+def clean_text(text):
+    """HTML 태그 제거 및 텍스트 정리"""
+    text = re.sub(r'\s+', ' ', text)
+    text = re.sub(r'<[^>]+>', '', text)
+    return text.strip()
+
+def get_article_content(url):
+    """URL에서 기사 내용 스크래핑"""
+    if not url or 'github.com' in url or 'twitter.com' in url:
+        return None
+        
+    try:
+        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'}
+        response = requests.get(url, headers=headers, timeout=10)
+        soup = BeautifulSoup(response.text, 'html.parser')
+        
+        # 불필요한 요소 제거
+        for tag in soup(['script', 'style', 'nav', 'footer', 'header']):
+            tag.decompose()
+            
+        paragraphs = soup.find_all('p')
+        text = ' '.join(p.get_text() for p in paragraphs)
+        text = clean_text(text)
+        
+        return text[:4000]  # 텍스트 길이 제한
+    except Exception as e:
+        print(f"Scraping error for {url}: {str(e)}")
+        return None
+
+def generate_summary(text):
+    """CohereForAI 모델을 사용한 요약 생성"""
+    if not text:
+        return None
+        
+    prompt = """Please analyze and summarize the following text in 2-3 sentences. 
+    Focus on the main points and key information:
+
+    Text: {text}
+    
+    Summary:"""
+    
+    try:
+        response = hf_client.text_generation(
+            prompt.format(text=text),
+            max_new_tokens=150,
+            temperature=0.7,
+            repetition_penalty=1.2
+        )
+        return response
+    except Exception as e:
+        print(f"Summary generation error: {str(e)}")
+        return None
+
+def process_hn_story(story, progress=None):
+    """개별 스토리 처리 및 요약"""
+    try:
+        url = story.get('url')
+        if not url:
+            return story, None
+            
+        content = get_article_content(url)
+        if not content:
+            return story, None
+            
+        summary_en = generate_summary(content)
+        if not summary_en:
+            return story, None
+            
+        summary_ko = translate_to_korean(summary_en)
+        return story, summary_ko
+        
+    except Exception as e:
+        print(f"Story processing error: {str(e)}")
+        return story, None
+
 def refresh_hn_stories():
-    """Hacker News 스토리 새로고침"""
+    """Hacker News 스토리 새로고침 (개선된 버전)"""
     status_msg = "Hacker News 포스트를 가져오는 중..."
-    
     outputs = [gr.update(value=status_msg, visible=True)]
     
-    # 모든 컴포넌트 초기화
-    for _ in hn_article_components:
+    # 컴포넌트 초기화
+    for comp in hn_article_components:
         outputs.extend([
             gr.update(visible=False),
             gr.update(),
             gr.update()
         ])
     
+    yield outputs
+    
     # 최신 스토리 가져오기
     stories = get_recent_stories()
     
-    # 결과 업데이트
-    outputs = [gr.update(value=f"총 {len(stories)}개의 포스트를 찾았습니다.", visible=True)]
+    # 병렬 처리로 요약 생성
+    processed_stories = []
+    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
+        future_to_story = {executor.submit(process_hn_story, story): story 
+                          for story in stories[:20]}  # 상위 20개만 처리
+        
+        for future in concurrent.futures.as_completed(future_to_story):
+            story, summary = future.result()
+            if summary:
+                processed_stories.append((story, summary))
+    
+    # 결과 정렬 및 출력
+    processed_stories.sort(key=lambda x: x[0].get('time', 0), reverse=True)
+    
+    outputs = [gr.update(value=f"총 {len(processed_stories)}개의 포스트가 처리되었습니다.", visible=True)]
     
     for idx, comp in enumerate(hn_article_components):
-        if idx < len(stories):
-            story = stories[idx]
+        if idx < len(processed_stories):
+            story, summary = processed_stories[idx]
             outputs.extend([
                 gr.update(visible=True),
                 gr.update(value=f"### [{story.get('title', 'Untitled')}]({story.get('url', '#')})"),
-                gr.update(value=f"**작성자:** {story.get('by', 'unknown')} | **시간:** {format_hn_time(story.get('time', 0))} | **점수:** {story.get('score', 0)} | **댓글:** {len(story.get('kids', []))}개")
+                gr.update(value=f"""
+                **작성자:** {story.get('by', 'unknown')} | 
+                **시간:** {format_hn_time(story.get('time', 0))} | 
+                **점수:** {story.get('score', 0)} | 
+                **댓글:** {len(story.get('kids', []))}개\n
+                **AI 요약:** {summary}
+                """)
             ])
         else:
             outputs.extend([
@@ -386,8 +495,7 @@ def refresh_hn_stories():
                 gr.update()
             ])
     
-    return outputs
-
+    yield outputs
         
 css = """
 footer {visibility: hidden;}
@@ -515,23 +623,24 @@ with gr.Blocks(theme="Nymbo/Nymbo_Theme", css=css, title="NewsAI 서비스") as
                                 'index': i,
                             })
 
+
         # AI 리포터 탭
         with gr.Tab("AI 리포터"):
-            gr.Markdown("지난 24시간 동안의 Hacker News 포스트를 보여줍니다.")
-            
+            gr.Markdown("지난 24시간 동안의 Hacker News 포스트를 AI가 요약하여 보여줍니다.")
+    
             with gr.Column():
                 refresh_button = gr.Button("새로고침", variant="primary")
                 status_message_hn = gr.Markdown("")
                 
                 with gr.Column(elem_id="hn_results_area"):
                     hn_articles_state = gr.State([])
-                    
+            
                     hn_article_components = []
-                    for i in range(100):
+                    for i in range(100):  # 상위 20개 포스트만 처리
                         with gr.Group(visible=False) as article_group:
                             title = gr.Markdown()
                             info = gr.Markdown()
-                            
+                    
                             hn_article_components.append({
                                 'group': article_group,
                                 'title': title,
@@ -541,9 +650,6 @@ with gr.Blocks(theme="Nymbo/Nymbo_Theme", css=css, title="NewsAI 서비스") as
 
 
 
-
-
-
     
 # 기존 함수들
     def search_and_display(query, country, articles_state, progress=gr.Progress()):