Update utils.py

utils.py

@@ -1,458 +1,459 @@
-# ==========================
-# Data Handling & Storage
-# ==========================
-import json
-import ast
-import pandas as pd
-import numpy as np
-
-# ==========================
-# Web Scraping & Data Retrieval
-# ==========================
-import requests
-import httpx
-import feedparser
-import concurrent.futures
-from bs4 import BeautifulSoup
-from googlesearch import search
-from urllib.parse import urlparse
-
-# ==========================
-# Natural Language Processing (NLP)
-# ==========================
-import nltk
-import spacy
-import gensim
-from nltk.corpus import stopwords
-from nltk.tokenize import word_tokenize
-from nltk.stem import WordNetLemmatizer
-from gensim.models import LdaModel
-from gensim.corpora import Dictionary
-from transformers import pipeline
-from deep_translator import GoogleTranslator
-from gtts import gTTS  # Text-to-speech
-
-# ==========================
-# Machine Learning & Text Analysis
-# ==========================
-from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer, ENGLISH_STOP_WORDS
-from sklearn.metrics.pairwise import cosine_similarity
-from sklearn.decomposition import NMF, LatentDirichletAllocation
-from sklearn.model_selection import RandomizedSearchCV
-
-# ==========================
-# Data Visualization
-# ==========================
-import matplotlib.pyplot as plt
-import seaborn as sns
-
-# ==========================
-# Utility & Performance Optimization
-# ==========================
-import re
-import os
-import io
-from collections import Counter
-from tqdm import tqdm  # progress bar
-
-
-def fetch_news_data(company_name: str, article_number: int):
-    excluded_domains = ["youtube.com", "en.wikipedia.org", "m.economictimes.com", "www.prnewswire.com", "economictimes.indiatimes.com", "www.moneycontrol.com"]
-
-    def is_valid_news_article(url, company_name):
-        try:
-            domain = urlparse(url).netloc  # extracts the domain
-            if company_name.lower() in domain.lower() or any(excluded_domain in domain for excluded_domain in excluded_domains):
-                return False
-            return True
-        except Exception:
-            return False  # handle unexpected errors
-
-    def get_top_articles(company_name, article_number):
-        query = f"{company_name} latest news article"
-        valid_urls = []
-
-        for url in search(query, num_results = article_number*2):
-            if is_valid_news_article(url, company_name):
-                valid_urls.append(url)
-            if len(valid_urls) > article_number+1:
-                break
-
-        return valid_urls
-
-    def extract_article_data(url):
-        headers = {
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
-        }
-
-        try:
-            response = requests.get(url, headers=headers)
-            response.raise_for_status()  # handle HTTP errors
-
-            soup = BeautifulSoup(response.content, 'html.parser')
-
-            # extract title
-            title = soup.title.string.strip() if soup.title else None
-            source = url.split('/')[2]  # Extract domain
-
-            # validate data
-            if not title:
-                return None
-
-            return {"title": title, "link": url, "source": source}
-
-        except (requests.exceptions.RequestException, AttributeError):
-            return None  # skip articles with invalid data
-
-    def main(company_name, article_number):
-        urls = get_top_articles(company_name, article_number)
-        # extract and validate article data
-        articles_data = [extract_article_data(url) for url in urls]
-        articles_data = [article for article in articles_data if article]  # remove None values
-
-        # create DataFrame only if valid articles exist
-        if articles_data:
-            df = pd.DataFrame(articles_data)
-        else:
-            df = pd.DataFrame(columns=["title", "link"])  # empty DataFrame if nothing was found
-
-        return df
-
-    df = main(company_name, article_number+1)
-    news_df_output = df[["title", "source"]].rename(columns={"title": "Headline", "source": "Source"})
-    news_df_output["Source"] = news_df_output["Source"].str.replace(r"^www\.", "", regex=True).str.split('.').str[0]
-    
-    yield {"news_df_output": news_df_output}
-
-    def get_article_text(url):
-        try:
-            headers = {'User-Agent': 'Mozilla/5.0'}
-            response = requests.get(url, headers=headers)
-            soup = BeautifulSoup(response.text, "html.parser")
-
-            # remove unwanted elements
-            for unwanted in soup.select("nav, aside, footer, header, .ad, .advertisement, .promo, .sidebar, .related-articles"):
-                unwanted.extract()
-
-            # try extracting from known article containers
-            article_body = soup.find(['article', 'div', 'section'], class_=['article-body', 'post-body', 'entry-content', 'main-content'])
-
-            if article_body:
-                paragraphs = article_body.find_all('p')
-                article_text = " ".join([p.get_text() for p in paragraphs]).strip()
-                return article_text if article_text else None  # return None if empty
-
-            # fallback to all <p> tags
-            paragraphs = soup.find_all('p')
-            article_text = " ".join([p.get_text() for p in paragraphs]).strip()
-
-            return article_text if article_text else None  # return None if empty
-
-        except Exception:
-            return None  # return None in case of an error
-    df['article_text'] = df['link'].apply(get_article_text)
-
-    df = df.reset_index(drop=True)
-
-    block_patterns = [
-        # Error messages (with variations)
-        r'Oops[!,\.]? something went wrong',
-        r'An error has occurred',
-        r'This content is not available',
-        r'Please enable JavaScript to continue',
-        r'Error loading content',
-        r'Follow Us',
-
-        # JavaScript patterns
-        r'var .*?;',
-        r'alert\(.*?\)',
-        r'console\.log\(.*?\)',
-        r'<script.*?</script>',
-        r'<noscript>.*?</noscript>',
-        r'<style.*?</style>',
-
-        # Loading or restricted content messages
-        r'Loading[\.]*',
-        r'You must be logged in to view this content',
-        r'This content is restricted',
-        r'Access denied',
-        r'Please disable your ad blocker',
-
-        # GDPR and cookie consent banners
-        r'This site uses cookies',
-        r'We use cookies to improve your experience',
-        r'By using this site, you agree to our use of cookies',
-        r'Accept Cookies',
-
-        # Stories or content teasers with any number
-        r'\d+\s*Stories',
-
-        # Miscellaneous
-        r'<iframe.*?</iframe>',
-        r'<meta.*?>',
-        r'<link.*?>',
-        r'Refresh the page and try again',
-        r'Click here if the page does not load',
-        r'© [0-9]{4}.*? All rights reserved',
-        r'Unauthorized access',
-        r'Terms of Service',
-        r'Privacy Policy',
-        r'<.*?>',
-        ]
-
-    pattern = '|'.join(block_patterns)
-    df['article_text'] = df['article_text'].str.replace(pattern, '', regex=True).str.strip()
-    df['article_text'] = df['article_text'].str.replace(r'\s+', ' ', regex=True).str.strip()
-
-    custom_stop_words = set(ENGLISH_STOP_WORDS.union({company_name.lower(), 'company', 'ttm', 'rs'}))
-
-    # add numeric values (integer, decimal, comma-separated, monetary)
-    numeric_patterns = re.compile(r'\b\d+(?:[\.,]\d+)?(?:,\d+)*\b|\$\d+(?:[\.,]\d+)?')
-    numeric_matches = set(re.findall(numeric_patterns, ' '.join(df['article_text'])))
-    custom_stop_words.update(numeric_matches)
-
-    # remove unwanted unicode characters (like \u2018, \u2019, etc.)
-    unicode_patterns = re.compile(r'[\u2018\u2019\u2020\u2021\u2014]')  # Add more if needed
-    df['article_text'] = df['article_text'].apply(lambda x: unicode_patterns.sub('', x))
-
-    custom_stop_words = list(custom_stop_words)
-
-    summarizer = pipeline("summarization", model="google/long-t5-tglobal-base")
-
-    def generate_summary(text):
-        try:
-            if len(text.split()) > 50:  # skip very short texts
-                summary = summarizer(text, max_length=150, min_length=50, do_sample=False)[0]['summary_text']
-                return summary
-            else:
-                return text
-        except Exception as e:
-            print(f"Error processing text: {e}")
-            return None
-
-    # apply summarization to the 'article_text' column
-    df['summary'] = df['article_text'].apply(generate_summary)
-
-    # load a pre-trained BERT-based sentiment model from Hugging Faces
-    sentiment_pipeline = pipeline("sentiment-analysis")
-
-    def analyze_sentiment(text):
-        """Analyze sentiment with a confidence-based neutral zone."""
-        if not text.strip():
-            return "Neutral"
-
-        try:
-            result = sentiment_pipeline(text)[0]
-            sentiment_label = result["label"]
-            confidence = round(result["score"], 2)
-
-            if confidence < 0.7:
-                return "Neutral"
-            return f"{sentiment_label.capitalize()} ({confidence})"
-        except Exception:
-            return "Error in sentiment analysis."
-
-    # apply sentiment analysis on the summary column
-    df['sentiment'] = df['summary'].apply(analyze_sentiment)
-
-    df['sentiment_label'] = df['sentiment'].str.extract(r'(Positive|Negative|Neutral)')
-
-    sentiment_bars = plt.figure(figsize=(7, 7))
-    sns.countplot(x=df['sentiment_label'], palette={'Positive': 'green', 'Negative': 'red', 'Neutral': 'gray'})
-    plt.title("Sentiment Analysis of Articles")
-    plt.xlabel("Sentiment")
-    plt.ylabel("Count")
-
-    # save the figure as an image file to use in gradio interface
-    sentiment_bars_file = "sentiment_bars.png"
-    sentiment_bars.savefig(sentiment_bars_file)
-    plt.close(sentiment_bars)
-
-    sentiment_counts = df['sentiment_label'].value_counts()
-
-    colors = {'Positive': 'green', 'Negative': 'red', 'Neutral': 'gray'}
-
-    sentiment_pie = plt.figure(figsize=(7, 7))
-    plt.pie(sentiment_counts, labels=sentiment_counts.index, autopct='%1.1f%%', colors=[colors[label] for label in sentiment_counts.index])
-    plt.title("Sentiment Distribution of Articles")
-
-    sentiment_pie_file = "sentiment_pie.png"
-    sentiment_pie.savefig(sentiment_pie_file)
-    plt.close(sentiment_pie)
-
-    df['combined_text'] = df['title'] + ' ' + df['summary'] # combine text for analysis
-
-    vectorizer = TfidfVectorizer(max_features=1000, stop_words=custom_stop_words)
-    tfidf = vectorizer.fit_transform(df['combined_text'])
-
-    n_topics = 5  # number of topics
-    nmf = NMF(n_components=n_topics, random_state=42)
-    W = nmf.fit_transform(tfidf)
-    H = nmf.components_
-
-    feature_names = vectorizer.get_feature_names_out()
-    topics = []
-    for topic_idx, topic in enumerate(H):
-        top_words = [feature_names[i] for i in topic.argsort()[-5:]][::-1]  # 5 words per topic
-        topics.append(", ".join(top_words))
-
-
-    def get_top_topics(row):
-        topic_indices = W[row].argsort()[-3:][::-1]  # get top 3 topics
-        return [topics[i] for i in topic_indices]
-
-    df['top_topics'] = [get_top_topics(i) for i in range(len(df))]
-    df['dominant_topic'] = W.argmax(axis=1)
-    df['topic_distribution'] = W.tolist()
-    similarity_matrix = cosine_similarity(W)
-
-    df['similarity_scores'] = similarity_matrix.mean(axis=1)
-    df['most_similar_article'] = similarity_matrix.argsort(axis=1)[:, -2]  # second highest value
-    df['least_similar_article'] = similarity_matrix.argsort(axis=1)[:, 0]  # lowest value
-
-    similarity_heatmap = plt.figure(figsize=(10, 8))
-    sns.heatmap(similarity_matrix, annot=True, fmt=".2f", cmap="coolwarm", xticklabels=False, yticklabels=False)
-    plt.title("Comparative Analysis of News Coverage Across Articles")
-
-    comparisons = []
-    for i in range(len(df)):
-        # find most similar and least similar articles
-        similar_idx = similarity_matrix[i].argsort()[-2]  # most similar (excluding itself)
-        least_similar_idx = similarity_matrix[i].argsort()[0]  # least similar
-
-        # build comparison text
-        comparison = {
-            "Most Similar": f"Article {i + 1} focuses on '{topics[df['dominant_topic'][i]]}', similar to Article {similar_idx + 1} which also discusses '{topics[df['dominant_topic'][similar_idx]]}'.",
-            "Least Similar": f"Article {i + 1} focuses on '{topics[df['dominant_topic'][i]]}', contrasting with Article {least_similar_idx + 1} which discusses '{topics[df['dominant_topic'][least_similar_idx]]}'."
-        }
-        comparisons.append(comparison)
-
-    df['coverage_comparison'] = comparisons
-    # find common and unique topics
-    all_topics = df['dominant_topic'].tolist()
-    topic_counter = Counter(all_topics)
-    common_topics = [topics[i] for i, count in topic_counter.items() if count > 1]
-    unique_topics = [topics[i] for i, count in topic_counter.items() if count == 1]
-
-    topic_overlap = {
-        "Common Topics": common_topics,
-        "Unique Topics": unique_topics
-    }
-    sentiment_counts = df['sentiment_label'].value_counts()
-    if sentiment_counts.get('Positive', 0) > sentiment_counts.get('Negative', 0):
-        sentiment = "Overall sentiment is positive."
-    elif sentiment_counts.get('Negative', 0) > sentiment_counts.get('Positive', 0):
-        sentiment = "Overall sentiment is negative."
-    else:
-        sentiment = "Overall sentiment is mixed."
-
-    def extract_relevant_topics(topics):
-        if isinstance(topics, str):
-            topics = ast.literal_eval(topics)  # convert string to list if needed
-
-        if len(topics) <= 2:
-            return topics
-
-        vectorizer = TfidfVectorizer()
-        tfidf_matrix = vectorizer.fit_transform(topics)
-        similarity_matrix = cosine_similarity(tfidf_matrix, tfidf_matrix)
-
-        # sum similarity scores for each topic
-        topic_scores = similarity_matrix.sum(axis=1)
-
-        # get top 2 highest scoring topics
-        top_indices = topic_scores.argsort()[-2:][::-1]
-        top_topics = [topics[i] for i in top_indices]
-
-        return top_topics
-
-
-    # ensure 'top_topics' is a list
-    df['top_topics'] = df['top_topics'].apply(lambda x: ast.literal_eval(x) if isinstance(x, str) else x)
-
-    # convert lists to sets for easy comparison
-    df['top_topics_set'] = df['top_topics'].apply(lambda x: set(x) if isinstance(x, list) else set())
-
-    # find common topics across all articles
-    if len(df) > 1:
-        common_topics = set.intersection(*df['top_topics_set'])
-    else:
-        common_topics = set()  # no common topics if only one article
-
-    # extract unique topics by removing common ones
-    df['unique_topics'] = df['top_topics_set'].apply(lambda x: list(x - common_topics) if x else [])
-
-    # drop the temporary 'top_topics_set' column
-    df.drop(columns=['top_topics_set'], inplace=True)
-
-
-    coverage_differences = []
-    for _, row in df.iterrows():
-        if row['most_similar_article'] in df.index and row['least_similar_article'] in df.index:
-            most_similar = df.loc[row['most_similar_article']]
-            least_similar = df.loc[row['least_similar_article']]
-
-            # extract most relevant topics
-            most_relevant_topics = extract_relevant_topics(row['top_topics'])
-            least_relevant_topics = extract_relevant_topics(least_similar['top_topics'])
-
-            if most_relevant_topics and least_relevant_topics:
-                comparison = {
-                    "Comparison": f"{row['title']} highlights {', '.join(row['top_topics'])}, while {most_similar['title']} discusses {', '.join(most_similar['top_topics'])}.",
-                    "Impact": f"The article emphasizes {most_relevant_topics[0]} and {most_relevant_topics[1]}, contrasting with {least_relevant_topics[0]} and {least_relevant_topics[1]} in the least similar article."
-                }
-                coverage_differences.append(comparison)
-    structured_summary = {
-        "Company": company_name,
-        "Articles": [
-            {
-                "Title": row['title'],
-                "Summary": row['summary'],
-                "Sentiment": row['sentiment'],
-                "Topics": row['top_topics'],
-                "Unique Topics": row['unique_topics']
-            }
-            for _, row in df.iterrows()
-        ],
-        "Comparative Sentiment Score": {
-            "Sentiment Distribution": df['sentiment'].value_counts().to_dict(),
-        },
-        "Topic Overlap": {
-            "Common Topics": list(common_topics) if common_topics else ["No common topics found"],
-            "Unique Topics": [
-                {"Title": row['title'], "Unique Topics": row['unique_topics']}
-                for _, row in df.iterrows()
-            ]
-        },
-        "Final Sentiment Analysis": f"{company_name}’s latest news coverage is mostly {df['sentiment'].mode()[0].lower()}. Potential market impact expected."
-    }
-
-    yield {"json_summary": structured_summary}
-    english_news = [f"Name of Company: {company_name}"]
-
-    for i, row in df.iterrows():
-        article_entry = f"Article {i + 1}: "
-        article_entry += f"{row['title']}; "
-        article_entry += f"Summary: {row['summary']} This article has a {row['sentiment_label'].lower()} sentiment."
-        english_news.append(article_entry)
-    yield {"english_news_list": english_news}
-    translator = GoogleTranslator(source='en', target='hi')  # 'hi' = Hindi
-
-    translated_news = []
-    for text in tqdm(english_news, desc="Translating"):
-        translated_news.append(translator.translate(text))
-    yield {"hindi_news_list": translated_news}
-    hindi_news = '; '.join(translated_news)
-    # yield {"hindi_news_text": hindi_news}
-    def text_to_speech(text, language='hi'):
-      tts = gTTS(text=text, lang=language, slow=False)
-      filename = "hindi_news.mp3"  # save file to path
-      tts.save(filename)
-      return filename
-    print(df)
-    news_audio = text_to_speech(hindi_news)
-    yield {"hindi_news_audio": news_audio}
-
-    yield {"bar_chart": sentiment_bars_file}
-
+# ==========================
+# Data Handling & Storage
+# ==========================
+import json
+import ast
+import pandas as pd
+import numpy as np
+
+# ==========================
+# Web Scraping & Data Retrieval
+# ==========================
+import requests
+import httpx
+import feedparser
+import concurrent.futures
+from bs4 import BeautifulSoup
+from googlesearch import search
+from urllib.parse import urlparse
+
+# ==========================
+# Natural Language Processing (NLP)
+# ==========================
+import nltk
+import spacy
+import gensim
+from nltk.corpus import stopwords
+from nltk.tokenize import word_tokenize
+from nltk.stem import WordNetLemmatizer
+from gensim.models import LdaModel
+from gensim.corpora import Dictionary
+from transformers import pipeline
+from deep_translator import GoogleTranslator
+from gtts import gTTS  # Text-to-speech
+
+# ==========================
+# Machine Learning & Text Analysis
+# ==========================
+from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer, ENGLISH_STOP_WORDS
+from sklearn.metrics.pairwise import cosine_similarity
+from sklearn.decomposition import NMF, LatentDirichletAllocation
+from sklearn.model_selection import RandomizedSearchCV
+
+# ==========================
+# Data Visualization
+# ==========================
+import matplotlib.pyplot as plt
+import seaborn as sns
+
+# ==========================
+# Utility & Performance Optimization
+# ==========================
+import re
+import os
+import io
+from collections import Counter
+from tqdm import tqdm  # progress bar
+
+
+def fetch_news_data(company_name: str, article_number: int):
+    excluded_domains = ["youtube.com", "en.wikipedia.org", "m.economictimes.com", "www.prnewswire.com", "economictimes.indiatimes.com", "www.moneycontrol.com"]
+
+    def is_valid_news_article(url, company_name):
+        try:
+            domain = urlparse(url).netloc  # extracts the domain
+            if company_name.lower() in domain.lower() or any(excluded_domain in domain for excluded_domain in excluded_domains):
+                return False
+            return True
+        except Exception:
+            return False  # handle unexpected errors
+
+    def get_top_articles(company_name, article_number):
+        query = f"{company_name} latest news article"
+        valid_urls = []
+
+        for url in search(query, num_results = article_number*2):
+            if is_valid_news_article(url, company_name):
+                valid_urls.append(url)
+            if len(valid_urls) > article_number+1:
+                break
+
+        return valid_urls
+
+    def extract_article_data(url):
+        headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
+        }
+
+        try:
+            response = requests.get(url, headers=headers)
+            response.raise_for_status()  # handle HTTP errors
+
+            soup = BeautifulSoup(response.content, 'html.parser')
+
+            # extract title
+            title = soup.title.string.strip() if soup.title else None
+            source = url.split('/')[2]  # Extract domain
+
+            # validate data
+            if not title:
+                return None
+
+            return {"title": title, "link": url, "source": source}
+
+        except (requests.exceptions.RequestException, AttributeError):
+            return None  # skip articles with invalid data
+
+    def main(company_name, article_number):
+        urls = get_top_articles(company_name, article_number)
+        # extract and validate article data
+        articles_data = [extract_article_data(url) for url in urls]
+        articles_data = [article for article in articles_data if article]  # remove None values
+
+        # create DataFrame only if valid articles exist
+        if articles_data:
+            df = pd.DataFrame(articles_data)
+        else:
+            df = pd.DataFrame(columns=["title", "link"])  # empty DataFrame if nothing was found
+
+        return df
+
+    df = main(company_name, article_number+1)
+    news_df_output = df[["title", "source"]].rename(columns={"title": "Headline", "source": "Source"})
+    news_df_output["Source"] = news_df_output["Source"].str.replace(r"^www\.", "", regex=True).str.split('.').str[0]
+    
+    yield {"news_df_output": news_df_output}
+
+    def get_article_text(url):
+        try:
+            headers = {'User-Agent': 'Mozilla/5.0'}
+            response = requests.get(url, headers=headers)
+            soup = BeautifulSoup(response.text, "html.parser")
+
+            # remove unwanted elements
+            for unwanted in soup.select("nav, aside, footer, header, .ad, .advertisement, .promo, .sidebar, .related-articles"):
+                unwanted.extract()
+
+            # try extracting from known article containers
+            article_body = soup.find(['article', 'div', 'section'], class_=['article-body', 'post-body', 'entry-content', 'main-content'])
+
+            if article_body:
+                paragraphs = article_body.find_all('p')
+                article_text = " ".join([p.get_text() for p in paragraphs]).strip()
+                return article_text if article_text else None  # return None if empty
+
+            # fallback to all <p> tags
+            paragraphs = soup.find_all('p')
+            article_text = " ".join([p.get_text() for p in paragraphs]).strip()
+
+            return article_text if article_text else None  # return None if empty
+
+        except Exception:
+            return None  # return None in case of an error
+    df['article_text'] = df['link'].apply(get_article_text)
+
+    df = df.reset_index(drop=True)
+
+    block_patterns = [
+        # Error messages (with variations)
+        r'Oops[!,\.]? something went wrong',
+        r'An error has occurred',
+        r'This content is not available',
+        r'Please enable JavaScript to continue',
+        r'Error loading content',
+        r'Follow Us',
+
+        # JavaScript patterns
+        r'var .*?;',
+        r'alert\(.*?\)',
+        r'console\.log\(.*?\)',
+        r'<script.*?</script>',
+        r'<noscript>.*?</noscript>',
+        r'<style.*?</style>',
+
+        # Loading or restricted content messages
+        r'Loading[\.]*',
+        r'You must be logged in to view this content',
+        r'This content is restricted',
+        r'Access denied',
+        r'Please disable your ad blocker',
+
+        # GDPR and cookie consent banners
+        r'This site uses cookies',
+        r'We use cookies to improve your experience',
+        r'By using this site, you agree to our use of cookies',
+        r'Accept Cookies',
+
+        # Stories or content teasers with any number
+        r'\d+\s*Stories',
+
+        # Miscellaneous
+        r'<iframe.*?</iframe>',
+        r'<meta.*?>',
+        r'<link.*?>',
+        r'Refresh the page and try again',
+        r'Click here if the page does not load',
+        r'© [0-9]{4}.*? All rights reserved',
+        r'Unauthorized access',
+        r'Terms of Service',
+        r'Privacy Policy',
+        r'<.*?>',
+        ]
+
+    pattern = '|'.join(block_patterns)
+    df['article_text'] = df['article_text'].str.replace(pattern, '', regex=True).str.strip()
+    df['article_text'] = df['article_text'].str.replace(r'\s+', ' ', regex=True).str.strip()
+
+    custom_stop_words = set(ENGLISH_STOP_WORDS.union({company_name.lower(), 'company', 'ttm', 'rs'}))
+
+    # add numeric values (integer, decimal, comma-separated, monetary)
+    numeric_patterns = re.compile(r'\b\d+(?:[\.,]\d+)?(?:,\d+)*\b|\$\d+(?:[\.,]\d+)?')
+    clean_text = ' '.join(df['article_text'].fillna('').astype(str))
+    numeric_matches = set(re.findall(numeric_patterns, clean_text))
+    custom_stop_words.update(numeric_matches)
+
+    # remove unwanted unicode characters (like \u2018, \u2019, etc.)
+    unicode_patterns = re.compile(r'[\u2018\u2019\u2020\u2021\u2014]')  # Add more if needed
+    df['article_text'] = df['article_text'].apply(lambda x: unicode_patterns.sub('', x))
+
+    custom_stop_words = list(custom_stop_words)
+
+    summarizer = pipeline("summarization", model="google/long-t5-tglobal-base")
+
+    def generate_summary(text):
+        try:
+            if len(text.split()) > 50:  # skip very short texts
+                summary = summarizer(text, max_length=150, min_length=50, do_sample=False)[0]['summary_text']
+                return summary
+            else:
+                return text
+        except Exception as e:
+            print(f"Error processing text: {e}")
+            return None
+
+    # apply summarization to the 'article_text' column
+    df['summary'] = df['article_text'].apply(generate_summary)
+
+    # load a pre-trained BERT-based sentiment model from Hugging Faces
+    sentiment_pipeline = pipeline("sentiment-analysis")
+
+    def analyze_sentiment(text):
+        """Analyze sentiment with a confidence-based neutral zone."""
+        if not text.strip():
+            return "Neutral"
+
+        try:
+            result = sentiment_pipeline(text)[0]
+            sentiment_label = result["label"]
+            confidence = round(result["score"], 2)
+
+            if confidence < 0.7:
+                return "Neutral"
+            return f"{sentiment_label.capitalize()} ({confidence})"
+        except Exception:
+            return "Error in sentiment analysis."
+
+    # apply sentiment analysis on the summary column
+    df['sentiment'] = df['summary'].apply(analyze_sentiment)
+
+    df['sentiment_label'] = df['sentiment'].str.extract(r'(Positive|Negative|Neutral)')
+
+    sentiment_bars = plt.figure(figsize=(7, 7))
+    sns.countplot(x=df['sentiment_label'], palette={'Positive': 'green', 'Negative': 'red', 'Neutral': 'gray'})
+    plt.title("Sentiment Analysis of Articles")
+    plt.xlabel("Sentiment")
+    plt.ylabel("Count")
+
+    # save the figure as an image file to use in gradio interface
+    sentiment_bars_file = "sentiment_bars.png"
+    sentiment_bars.savefig(sentiment_bars_file)
+    plt.close(sentiment_bars)
+
+    sentiment_counts = df['sentiment_label'].value_counts()
+
+    colors = {'Positive': 'green', 'Negative': 'red', 'Neutral': 'gray'}
+
+    sentiment_pie = plt.figure(figsize=(7, 7))
+    plt.pie(sentiment_counts, labels=sentiment_counts.index, autopct='%1.1f%%', colors=[colors[label] for label in sentiment_counts.index])
+    plt.title("Sentiment Distribution of Articles")
+
+    sentiment_pie_file = "sentiment_pie.png"
+    sentiment_pie.savefig(sentiment_pie_file)
+    plt.close(sentiment_pie)
+
+    df['combined_text'] = df['title'] + ' ' + df['summary'] # combine text for analysis
+
+    vectorizer = TfidfVectorizer(max_features=1000, stop_words=custom_stop_words)
+    tfidf = vectorizer.fit_transform(df['combined_text'])
+
+    n_topics = 5  # number of topics
+    nmf = NMF(n_components=n_topics, random_state=42)
+    W = nmf.fit_transform(tfidf)
+    H = nmf.components_
+
+    feature_names = vectorizer.get_feature_names_out()
+    topics = []
+    for topic_idx, topic in enumerate(H):
+        top_words = [feature_names[i] for i in topic.argsort()[-5:]][::-1]  # 5 words per topic
+        topics.append(", ".join(top_words))
+
+
+    def get_top_topics(row):
+        topic_indices = W[row].argsort()[-3:][::-1]  # get top 3 topics
+        return [topics[i] for i in topic_indices]
+
+    df['top_topics'] = [get_top_topics(i) for i in range(len(df))]
+    df['dominant_topic'] = W.argmax(axis=1)
+    df['topic_distribution'] = W.tolist()
+    similarity_matrix = cosine_similarity(W)
+
+    df['similarity_scores'] = similarity_matrix.mean(axis=1)
+    df['most_similar_article'] = similarity_matrix.argsort(axis=1)[:, -2]  # second highest value
+    df['least_similar_article'] = similarity_matrix.argsort(axis=1)[:, 0]  # lowest value
+
+    similarity_heatmap = plt.figure(figsize=(10, 8))
+    sns.heatmap(similarity_matrix, annot=True, fmt=".2f", cmap="coolwarm", xticklabels=False, yticklabels=False)
+    plt.title("Comparative Analysis of News Coverage Across Articles")
+
+    comparisons = []
+    for i in range(len(df)):
+        # find most similar and least similar articles
+        similar_idx = similarity_matrix[i].argsort()[-2]  # most similar (excluding itself)
+        least_similar_idx = similarity_matrix[i].argsort()[0]  # least similar
+
+        # build comparison text
+        comparison = {
+            "Most Similar": f"Article {i + 1} focuses on '{topics[df['dominant_topic'][i]]}', similar to Article {similar_idx + 1} which also discusses '{topics[df['dominant_topic'][similar_idx]]}'.",
+            "Least Similar": f"Article {i + 1} focuses on '{topics[df['dominant_topic'][i]]}', contrasting with Article {least_similar_idx + 1} which discusses '{topics[df['dominant_topic'][least_similar_idx]]}'."
+        }
+        comparisons.append(comparison)
+
+    df['coverage_comparison'] = comparisons
+    # find common and unique topics
+    all_topics = df['dominant_topic'].tolist()
+    topic_counter = Counter(all_topics)
+    common_topics = [topics[i] for i, count in topic_counter.items() if count > 1]
+    unique_topics = [topics[i] for i, count in topic_counter.items() if count == 1]
+
+    topic_overlap = {
+        "Common Topics": common_topics,
+        "Unique Topics": unique_topics
+    }
+    sentiment_counts = df['sentiment_label'].value_counts()
+    if sentiment_counts.get('Positive', 0) > sentiment_counts.get('Negative', 0):
+        sentiment = "Overall sentiment is positive."
+    elif sentiment_counts.get('Negative', 0) > sentiment_counts.get('Positive', 0):
+        sentiment = "Overall sentiment is negative."
+    else:
+        sentiment = "Overall sentiment is mixed."
+
+    def extract_relevant_topics(topics):
+        if isinstance(topics, str):
+            topics = ast.literal_eval(topics)  # convert string to list if needed
+
+        if len(topics) <= 2:
+            return topics
+
+        vectorizer = TfidfVectorizer()
+        tfidf_matrix = vectorizer.fit_transform(topics)
+        similarity_matrix = cosine_similarity(tfidf_matrix, tfidf_matrix)
+
+        # sum similarity scores for each topic
+        topic_scores = similarity_matrix.sum(axis=1)
+
+        # get top 2 highest scoring topics
+        top_indices = topic_scores.argsort()[-2:][::-1]
+        top_topics = [topics[i] for i in top_indices]
+
+        return top_topics
+
+
+    # ensure 'top_topics' is a list
+    df['top_topics'] = df['top_topics'].apply(lambda x: ast.literal_eval(x) if isinstance(x, str) else x)
+
+    # convert lists to sets for easy comparison
+    df['top_topics_set'] = df['top_topics'].apply(lambda x: set(x) if isinstance(x, list) else set())
+
+    # find common topics across all articles
+    if len(df) > 1:
+        common_topics = set.intersection(*df['top_topics_set'])
+    else:
+        common_topics = set()  # no common topics if only one article
+
+    # extract unique topics by removing common ones
+    df['unique_topics'] = df['top_topics_set'].apply(lambda x: list(x - common_topics) if x else [])
+
+    # drop the temporary 'top_topics_set' column
+    df.drop(columns=['top_topics_set'], inplace=True)
+
+
+    coverage_differences = []
+    for _, row in df.iterrows():
+        if row['most_similar_article'] in df.index and row['least_similar_article'] in df.index:
+            most_similar = df.loc[row['most_similar_article']]
+            least_similar = df.loc[row['least_similar_article']]
+
+            # extract most relevant topics
+            most_relevant_topics = extract_relevant_topics(row['top_topics'])
+            least_relevant_topics = extract_relevant_topics(least_similar['top_topics'])
+
+            if most_relevant_topics and least_relevant_topics:
+                comparison = {
+                    "Comparison": f"{row['title']} highlights {', '.join(row['top_topics'])}, while {most_similar['title']} discusses {', '.join(most_similar['top_topics'])}.",
+                    "Impact": f"The article emphasizes {most_relevant_topics[0]} and {most_relevant_topics[1]}, contrasting with {least_relevant_topics[0]} and {least_relevant_topics[1]} in the least similar article."
+                }
+                coverage_differences.append(comparison)
+    structured_summary = {
+        "Company": company_name,
+        "Articles": [
+            {
+                "Title": row['title'],
+                "Summary": row['summary'],
+                "Sentiment": row['sentiment'],
+                "Topics": row['top_topics'],
+                "Unique Topics": row['unique_topics']
+            }
+            for _, row in df.iterrows()
+        ],
+        "Comparative Sentiment Score": {
+            "Sentiment Distribution": df['sentiment'].value_counts().to_dict(),
+        },
+        "Topic Overlap": {
+            "Common Topics": list(common_topics) if common_topics else ["No common topics found"],
+            "Unique Topics": [
+                {"Title": row['title'], "Unique Topics": row['unique_topics']}
+                for _, row in df.iterrows()
+            ]
+        },
+        "Final Sentiment Analysis": f"{company_name}’s latest news coverage is mostly {df['sentiment'].mode()[0].lower()}. Potential market impact expected."
+    }
+
+    yield {"json_summary": structured_summary}
+    english_news = [f"Name of Company: {company_name}"]
+
+    for i, row in df.iterrows():
+        article_entry = f"Article {i + 1}: "
+        article_entry += f"{row['title']}; "
+        article_entry += f"Summary: {row['summary']} This article has a {row['sentiment_label'].lower()} sentiment."
+        english_news.append(article_entry)
+    yield {"english_news_list": english_news}
+    translator = GoogleTranslator(source='en', target='hi')  # 'hi' = Hindi
+
+    translated_news = []
+    for text in tqdm(english_news, desc="Translating"):
+        translated_news.append(translator.translate(text))
+    yield {"hindi_news_list": translated_news}
+    hindi_news = '; '.join(translated_news)
+    # yield {"hindi_news_text": hindi_news}
+    def text_to_speech(text, language='hi'):
+      tts = gTTS(text=text, lang=language, slow=False)
+      filename = "hindi_news.mp3"  # save file to path
+      tts.save(filename)
+      return filename
+    print(df)
+    news_audio = text_to_speech(hindi_news)
+    yield {"hindi_news_audio": news_audio}
+
+    yield {"bar_chart": sentiment_bars_file}
+
     yield {"pie_chart": sentiment_pie_file}