debugging

app.py

@@ -62,7 +62,6 @@ if submitted:
         try:
             spinner_box.markdown("⏳ Running analysis pipeline...")
 
-            # Run the full pipeline
             html_paths, articles_df, insights_df = run_pipeline(csv_path, tavily_api_key, progress_callback=log)
 
             spinner_box.success("✅ Analysis complete!")
@@ -79,15 +78,29 @@ if submitted:
 
             # --- Articles Tab ---
             with tab_articles:
+                st.subheader("📋 Articles Table")
                 if not articles_df.empty:
                     st.dataframe(articles_df, use_container_width=True)
+                    st.download_button(
+                        label="⬇️ Download Articles CSV",
+                        data=articles_df.to_csv(index=False).encode("utf-8"),
+                        file_name="articles.csv",
+                        mime="text/csv"
+                    )
                 else:
                     st.info("No articles available.")
 
             # --- Insights Tab ---
             with tab_insights:
+                st.subheader("📊 Investment Insights")
                 if not insights_df.empty:
                     st.dataframe(insights_df, use_container_width=True)
+                    st.download_button(
+                        label="⬇️ Download Insights CSV",
+                        data=insights_df.to_csv(index=False).encode("utf-8"),
+                        file_name="insights.csv",
+                        mime="text/csv"
+                    )
                 else:
                     st.info("No insights available.")
 
@@ -95,16 +108,11 @@ if submitted:
             spinner_box.error("❌ Failed.")
             log_box.error(f"❌ Error: {e}")
 
-
 # import os
 # import sys
 # import tempfile
-# import time
-# import itertools
 # import streamlit as st
 # import pandas as pd
-# from threading import Thread
-# from io import StringIO
 
 # # Add 'src' to Python path so we can import main.py
 # sys.path.append(os.path.join(os.path.dirname(__file__), 'src'))
@@ -121,7 +129,6 @@ if submitted:
 # # === Topic Input ===
 # st.subheader("📈 Topics of Interest")
 # topics_data = []
-
 # with st.form("topics_form"):
 #     topic_count = st.number_input("How many topics?", min_value=1, max_value=10, value=1, step=1)
 
@@ -135,6 +142,12 @@ if submitted:
 
 #     submitted = st.form_submit_button("Run Analysis")
 
+# # === Tabs Setup ===
+# tab_report, tab_articles, tab_insights = st.tabs(["📝 Report", "📋 Articles", "📊 Insights"])
+# articles_df = pd.DataFrame()
+# insights_df = pd.DataFrame()
+# html_paths = []
+
 # # === Submission logic ===
 # if submitted:
 #     if not openai_api_key or not tavily_api_key or not all([td['topic'] for td in topics_data]):
@@ -148,77 +161,48 @@ if submitted:
 #             df.to_csv(tmp_csv.name, index=False)
 #             csv_path = tmp_csv.name
 
-#         # === UI Elements ===
-#         spinner_box = st.empty()   # For rotating messages
-#         log_box = st.empty()       # For logs
+#         spinner_box = st.empty()
+#         log_box = st.empty()
 #         logs = []
-#         rotating = True
 
 #         def log(msg):
 #             logs.append(msg)
 #             log_box.code("\n".join(logs))
 
-#         # === Rotating UI Messages ===
-#         def rotating_messages():
-#             messages = itertools.cycle([
-#                 "🔍 Searching financial news...",
-#                 "🧠 Running language models...",
-#                 "📊 Analyzing investor sentiment...",
-#                 "📝 Summarizing key takeaways...",
-#                 "💹 Building markdown reports..."
-#             ])
-#             while rotating:
-#                 spinner_box.markdown(f"⏳ {next(messages)}")
-#                 time.sleep(1.5)
-
-#         rotator_thread = Thread(target=rotating_messages)
-#         rotator_thread.start()
-
 #         try:
-#             # Check API Keys
-#             import openai
-#             openai.OpenAI(api_key=openai_api_key).models.list()
-#             log("✅ OpenAI API key is valid.")
-
-#             import requests
-#             tavily_test = requests.post(
-#                 "https://api.tavily.com/search",
-#                 headers={"Authorization": f"Bearer {tavily_api_key}"},
-#                 json={"query": "test", "days": 1, "max_results": 1}
-#             )
-#             if tavily_test.status_code == 200:
-#                 log("✅ Tavily API key is valid.")
-#             else:
-#                 raise ValueError(f"Tavily error: {tavily_test.status_code} - {tavily_test.text}")
+#             spinner_box.markdown("⏳ Running analysis pipeline...")
 
 #             # Run the full pipeline
-#             log("🚀 Running analysis pipeline...")
-#             output_path = run_pipeline(csv_path, tavily_api_key, progress_callback=log)
+#             html_paths, articles_df, insights_df = run_pipeline(csv_path, tavily_api_key, progress_callback=log)
 
-#             rotating = False
-#             rotator_thread.join()
 #             spinner_box.success("✅ Analysis complete!")
 
-#             if output_path and isinstance(output_path, list):
-#                 for path in output_path:
-#                     if os.path.exists(path):
-#                         with open(path, 'r', encoding='utf-8') as file:
-#                             html_content = file.read()
-#                             filename = os.path.basename(path)
-
-#                             st.download_button(
-#                                 label=f"📥 Download {filename}",
-#                                 data=html_content,
-#                                 file_name=filename,
-#                                 mime="text/html"
-#                             )
+#             # --- Report Tab ---
+#             with tab_report:
+#                 if html_paths:
+#                     for path in html_paths:
+#                         with open(path, 'r', encoding='utf-8') as f:
+#                             html_content = f.read()
 #                             st.components.v1.html(html_content, height=600, scrolling=True)
-#             else:
-#                 st.error("❌ No reports were generated.")
+#                 else:
+#                     st.error("❌ No reports were generated.")
+
+#             # --- Articles Tab ---
+#             with tab_articles:
+#                 if not articles_df.empty:
+#                     st.dataframe(articles_df, use_container_width=True)
+#                 else:
+#                     st.info("No articles available.")
+
+#             # --- Insights Tab ---
+#             with tab_insights:
+#                 if not insights_df.empty:
+#                     st.dataframe(insights_df, use_container_width=True)
+#                 else:
+#                     st.info("No insights available.")
 
 #         except Exception as e:
-#             rotating = False
-#             rotator_thread.join()
 #             spinner_box.error("❌ Failed.")
 #             log_box.error(f"❌ Error: {e}")
 
+

src/main.py

@@ -2,11 +2,8 @@ import os
 import pandas as pd
 from datetime import datetime
 from dotenv import load_dotenv
-import traceback
-
 from md_html import convert_single_md_to_html as convert_md_to_html
 from news_analysis import fetch_deep_news, generate_value_investor_report
-from csv_utils import detect_changes
 from fin_interpreter import analyze_article
 
 BASE_DIR = os.path.dirname(os.path.dirname(__file__))
@@ -30,30 +27,33 @@ def build_metrics_box(topic, num_articles):
 """
 
 
-def run_value_investing_analysis(csv_path, progress_callback=None):
-    """
-    Runs the analysis for all topics in the CSV.
-    Returns:
-        md_files (list of md file paths)
-        all_articles (list of article dicts)
-    """
-    current_df = pd.read_csv(csv_path)
-    prev_path = os.path.join(BASE_DIR, "investing_topics_prev.csv")
+def derive_priority(sentiment, confidence):
+    """Basic logic to derive priority for articles."""
+    if sentiment == "Positive" and confidence > 0.75:
+        return "High"
+    elif sentiment == "Negative" and confidence > 0.6:
+        return "High"
+    elif confidence > 0.5:
+        return "Medium"
+    return "Low"
 
-    if os.path.exists(prev_path):
-        previous_df = pd.read_csv(prev_path)
-        changed_df = detect_changes(current_df, previous_df)
-        if changed_df.empty:
-            if progress_callback:
-                progress_callback("✅ No changes detected. Skipping processing.")
-            return [], []
+
+def derive_signal(sentiment, confidence):
+    """Basic investment signal logic."""
+    if sentiment == "Positive" and confidence > 0.7:
+        return "Buy"
+    elif sentiment == "Negative":
+        return "Avoid"
     else:
-        changed_df = current_df
+        return "Watch"
 
+
+def run_value_investing_analysis(csv_path, progress_callback=None):
+    current_df = pd.read_csv(csv_path)
     new_md_files = []
     all_articles = []
 
-    for _, row in changed_df.iterrows():
+    for _, row in current_df.iterrows():
         topic = row.get("topic")
         timespan = row.get("timespan_days", 7)
         msg = f"🔍 Processing: {topic} ({timespan} days)"
@@ -64,38 +64,46 @@ def run_value_investing_analysis(csv_path, progress_callback=None):
         news = fetch_deep_news(topic, timespan)
         if not news:
             warning = f"⚠️ No news found for: {topic}"
-            print(warning)
             if progress_callback:
                 progress_callback(warning)
             continue
 
-        # Add articles to all_articles
+        # Process each article
         for article in news:
+            summary = article.get("summary", "")
+            title = article.get("title", "Untitled")
+            url = article.get("url", "")
+            date = article.get("date", datetime.now().strftime("%Y-%m-%d"))
+            company = article.get("company", topic)
+
             try:
-                res = analyze_article(article.get("summary", ""))
+                res = analyze_article(summary)
                 if isinstance(res, dict):
-                    sentiment = res.get("sentiment")
-                    confidence = res.get("confidence")
-                    signal = res.get("signal")
+                    sentiment = res.get("sentiment", "Neutral")
+                    confidence = float(res.get("confidence", 0.0))
+                    signal = res.get("signal", "Watch")
                 else:
-                    sentiment, confidence, signal = res[0], res[1], res[2]
-            except Exception as e:
-                sentiment, confidence, signal = "Unknown", 0.0, "None"
-                print(f"Error analyzing article: {e}")
+                    sentiment, confidence, signal = res[0], float(res[1]), res[2]
+            except Exception:
+                sentiment, confidence, signal = "Neutral", 0.0, "Watch"
+
+            priority = derive_priority(sentiment, confidence)
+            if signal == "None":
+                signal = derive_signal(sentiment, confidence)
 
             all_articles.append({
-                "Title": article.get("title"),
-                "URL": article.get("url"),
-                "Summary": article.get("summary"),
-                "Priority": article.get("priority", "Low"),
-                "Date": article.get("date"),
-                "Company": article.get("company", topic),
+                "Title": title,
+                "URL": url,
+                "Summary": summary,
+                "Priority": priority,
+                "Date": date,
+                "Company": company,
                 "Sentiment": sentiment,
-                "Confidence": confidence,
+                "Confidence": round(confidence, 2),
                 "Signal": signal
             })
 
-        # Generate report
+        # Generate markdown report
         report_body = generate_value_investor_report(topic, news)
         metrics_md = build_metrics_box(topic, len(news))
         full_md = metrics_md + report_body
@@ -113,12 +121,37 @@ def run_value_investing_analysis(csv_path, progress_callback=None):
 
         new_md_files.append(filepath)
 
-    if progress_callback:
-        progress_callback(f"✅ Markdown saved to: {DATA_DIR}")
-    current_df.to_csv(prev_path, index=False)
     return new_md_files, all_articles
 
 
+def build_company_insights(articles_df):
+    if articles_df.empty:
+        return pd.DataFrame()
+
+    insights = []
+    for company, group in articles_df.groupby("Company"):
+        mentions = len(group)
+        dominant_sentiment = group["Sentiment"].mode()[0] if not group["Sentiment"].mode().empty else "Neutral"
+        dominant_signal = group["Signal"].mode()[0] if not group["Signal"].mode().empty else "Watch"
+        avg_confidence = round(group["Confidence"].mean(), 2)
+        risk_level = "High" if (dominant_sentiment == "Negative" and avg_confidence > 0.5) else "Low"
+        if dominant_sentiment == "Neutral":
+            risk_level = "Medium"
+
+        highlights = " | ".join(group["Summary"].head(2).tolist())
+        insights.append({
+            "Company": company,
+            "Mentions": mentions,
+            "Sentiment": dominant_sentiment,
+            "Signal": dominant_signal,
+            "Risk": risk_level,
+            "Confidence": avg_confidence,
+            "Highlights": highlights
+        })
+
+    return pd.DataFrame(insights)
+
+
 def run_pipeline(csv_path, tavily_api_key, progress_callback=None):
     os.environ["TAVILY_API_KEY"] = tavily_api_key
 
@@ -134,39 +167,24 @@ def run_pipeline(csv_path, tavily_api_key, progress_callback=None):
     return new_html_paths, articles_df, insights_df
 
 
-def build_company_insights(articles_df):
-    if articles_df.empty:
-        return pd.DataFrame()
-    grouped = (
-        articles_df.groupby("Company")
-        .agg({
-            "Title": "count",
-            "Sentiment": lambda x: x.mode()[0] if not x.mode().empty else "Neutral",
-            "Signal": lambda x: x.mode()[0] if not x.mode().empty else "Watch"
-        })
-        .reset_index()
-        .rename(columns={"Title": "Mentions"})
-    )
-    return grouped
-
-
 if __name__ == "__main__":
-    md_files, _ = run_value_investing_analysis(CSV_PATH)
+    md_files, all_articles = run_value_investing_analysis(CSV_PATH)
     for md in md_files:
         convert_md_to_html(md, HTML_DIR)
     print(f"🌐 All reports converted to HTML at: {HTML_DIR}")
 
-#import os
-# import sys
+
+# import os
+# import pandas as pd
 # from datetime import datetime
 # from dotenv import load_dotenv
-# import pandas as pd
+# import traceback
 
 # from md_html import convert_single_md_to_html as convert_md_to_html
 # from news_analysis import fetch_deep_news, generate_value_investor_report
 # from csv_utils import detect_changes
+# from fin_interpreter import analyze_article
 
-# # === Setup Paths ===
 # BASE_DIR = os.path.dirname(os.path.dirname(__file__))
 # DATA_DIR = os.path.join(BASE_DIR, "data")
 # HTML_DIR = os.path.join(BASE_DIR, "html")
@@ -175,9 +193,9 @@ if __name__ == "__main__":
 # os.makedirs(DATA_DIR, exist_ok=True)
 # os.makedirs(HTML_DIR, exist_ok=True)
 
-# # === Load .env ===
 # load_dotenv()
 
+
 # def build_metrics_box(topic, num_articles):
 #     now = datetime.now().strftime("%Y-%m-%d %H:%M")
 #     return f"""
@@ -187,21 +205,29 @@ if __name__ == "__main__":
 # >
 # """
 
+
 # def run_value_investing_analysis(csv_path, progress_callback=None):
+#     """
+#     Runs the analysis for all topics in the CSV.
+#     Returns:
+#         md_files (list of md file paths)
+#         all_articles (list of article dicts)
+#     """
 #     current_df = pd.read_csv(csv_path)
 #     prev_path = os.path.join(BASE_DIR, "investing_topics_prev.csv")
-    
+
 #     if os.path.exists(prev_path):
 #         previous_df = pd.read_csv(prev_path)
 #         changed_df = detect_changes(current_df, previous_df)
 #         if changed_df.empty:
 #             if progress_callback:
 #                 progress_callback("✅ No changes detected. Skipping processing.")
-#             return []
+#             return [], []
 #     else:
 #         changed_df = current_df
 
 #     new_md_files = []
+#     all_articles = []
 
 #     for _, row in changed_df.iterrows():
 #         topic = row.get("topic")
@@ -219,20 +245,42 @@ if __name__ == "__main__":
 #                 progress_callback(warning)
 #             continue
 
+#         # Add articles to all_articles
+#         for article in news:
+#             try:
+#                 res = analyze_article(article.get("summary", ""))
+#                 if isinstance(res, dict):
+#                     sentiment = res.get("sentiment")
+#                     confidence = res.get("confidence")
+#                     signal = res.get("signal")
+#                 else:
+#                     sentiment, confidence, signal = res[0], res[1], res[2]
+#             except Exception as e:
+#                 sentiment, confidence, signal = "Unknown", 0.0, "None"
+#                 print(f"Error analyzing article: {e}")
+
+#             all_articles.append({
+#                 "Title": article.get("title"),
+#                 "URL": article.get("url"),
+#                 "Summary": article.get("summary"),
+#                 "Priority": article.get("priority", "Low"),
+#                 "Date": article.get("date"),
+#                 "Company": article.get("company", topic),
+#                 "Sentiment": sentiment,
+#                 "Confidence": confidence,
+#                 "Signal": signal
+#             })
+
+#         # Generate report
 #         report_body = generate_value_investor_report(topic, news)
-#         image_url = "https://via.placeholder.com/1281x721?text=No+Image+Available"
-#         image_credit = "Image placeholder"
-
 #         metrics_md = build_metrics_box(topic, len(news))
 #         full_md = metrics_md + report_body
 
-#         base_filename = f"{topic.replace(' ', '_').lower()}_{datetime.now().strftime('%Y-%m-%d')}"
-#         filename = base_filename + ".md"
+#         filename = f"{topic.replace(' ', '_').lower()}_{datetime.now().strftime('%Y-%m-%d')}.md"
 #         filepath = os.path.join(DATA_DIR, filename)
-
 #         counter = 1
 #         while os.path.exists(filepath):
-#             filename = f"{base_filename}_{counter}.md"
+#             filename = f"{topic.replace(' ', '_').lower()}_{datetime.now().strftime('%Y-%m-%d')}_{counter}.md"
 #             filepath = os.path.join(DATA_DIR, filename)
 #             counter += 1
 
@@ -244,25 +292,43 @@ if __name__ == "__main__":
 #     if progress_callback:
 #         progress_callback(f"✅ Markdown saved to: {DATA_DIR}")
 #     current_df.to_csv(prev_path, index=False)
-#     return new_md_files
+#     return new_md_files, all_articles
+
 
 # def run_pipeline(csv_path, tavily_api_key, progress_callback=None):
 #     os.environ["TAVILY_API_KEY"] = tavily_api_key
 
-#     new_md_files = run_value_investing_analysis(csv_path, progress_callback)
+#     new_md_files, all_articles = run_value_investing_analysis(csv_path, progress_callback)
 #     new_html_paths = []
-
 #     for md_path in new_md_files:
 #         convert_md_to_html(md_path, HTML_DIR)
 #         html_path = os.path.join(HTML_DIR, os.path.basename(md_path).replace(".md", ".html"))
 #         new_html_paths.append(html_path)
 
-#     return new_html_paths
+#     articles_df = pd.DataFrame(all_articles)
+#     insights_df = build_company_insights(articles_df)
+#     return new_html_paths, articles_df, insights_df
+
+
+# def build_company_insights(articles_df):
+#     if articles_df.empty:
+#         return pd.DataFrame()
+#     grouped = (
+#         articles_df.groupby("Company")
+#         .agg({
+#             "Title": "count",
+#             "Sentiment": lambda x: x.mode()[0] if not x.mode().empty else "Neutral",
+#             "Signal": lambda x: x.mode()[0] if not x.mode().empty else "Watch"
+#         })
+#         .reset_index()
+#         .rename(columns={"Title": "Mentions"})
+#     )
+#     return grouped
+
 
 # if __name__ == "__main__":
-#     md_files = run_value_investing_analysis(CSV_PATH)
+#     md_files, _ = run_value_investing_analysis(CSV_PATH)
 #     for md in md_files:
 #         convert_md_to_html(md, HTML_DIR)
 #     print(f"🌐 All reports converted to HTML at: {HTML_DIR}")
 
-