Update Linkedin_Data_API_Calls.py

Linkedin_Data_API_Calls.py

@@ -1,8 +1,10 @@
 import json
 import requests
 import html
+import time # Added for potential rate limiting if needed
 from datetime import datetime
 from collections import defaultdict
+from urllib.parse import quote # Added for URL encoding
 from transformers import pipeline
 
 from sessions import create_session
@@ -14,19 +16,72 @@ import logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
 API_V2_BASE = 'https://api.linkedin.com/v2'
-API_REST_BASE = "https://api.linkedin.com/rest" # Corrected from API_REST_BASE to API_REST_BASE
+API_REST_BASE = "https://api.linkedin.com/rest"
 
-# Initialize sentiment pipeline (consider loading it once globally if this module is imported multiple times)
+# Initialize sentiment pipeline (loaded once globally)
 sentiment_pipeline = pipeline("text-classification", model="tabularisai/multilingual-sentiment-analysis")
 
+# --- Utility Function ---
+def extract_text_from_mention_commentary(commentary):
+    """
+    Extracts clean text from a commentary string, removing potential placeholders like {mention}.
+    """
+    import re 
+    if not commentary:
+        return ""
+    return re.sub(r"{.*?}", "", commentary).strip()
+
+# --- Core Sentiment Analysis Helper ---
+def _get_sentiment_from_text(text_to_analyze):
+    """
+    Analyzes a single piece of text and returns its sentiment label and raw counts.
+    Returns a dict: {"label": "Sentiment Label", "counts": defaultdict(int)}
+    """
+    sentiment_counts = defaultdict(int)
+    dominant_sentiment_label = "Neutral 😐" # Default
+    
+    if not text_to_analyze or not text_to_analyze.strip():
+        return {"label": dominant_sentiment_label, "counts": sentiment_counts}
+
+    try:
+        # Truncate to avoid issues with very long texts for the model
+        analysis_result = sentiment_pipeline(str(text_to_analyze)[:512]) 
+        label = analysis_result[0]['label'].upper()
+        
+        if label in ['POSITIVE', 'VERY POSITIVE']:
+            dominant_sentiment_label = 'Positive 👍'
+            sentiment_counts['Positive 👍'] += 1
+        elif label in ['NEGATIVE', 'VERY NEGATIVE']:
+            dominant_sentiment_label = 'Negative 👎'
+            sentiment_counts['Negative 👎'] += 1
+        elif label == 'NEUTRAL':
+            dominant_sentiment_label = 'Neutral 😐' # Already default, but for clarity
+            sentiment_counts['Neutral 😐'] += 1
+        else:
+            dominant_sentiment_label = 'Unknown' # Catch any other labels from the model
+            sentiment_counts['Unknown'] += 1
+        
+    except Exception as e:
+        # Log the error with more context if possible
+        logging.error(f"Sentiment analysis failed for text snippet '{str(text_to_analyze)[:50]}...'. Error: {e}")
+        sentiment_counts['Error'] += 1
+        dominant_sentiment_label = "Error" # Indicate error in sentiment
+
+    return {"label": dominant_sentiment_label, "counts": sentiment_counts}
+
+
+# --- Post Retrieval Functions ---
 def fetch_linkedin_posts_core(comm_client_id, community_token, org_urn, count):
     """
     Fetches raw posts, their basic statistics, and performs summarization/categorization.
-    Does NOT fetch comments or analyze sentiment.
+    Does NOT fetch comments or analyze sentiment of comments here.
     """
     token_dict = community_token if isinstance(community_token, dict) else {'access_token': community_token, 'token_type': 'Bearer'}
     session = create_session(comm_client_id, token=token_dict)
-    org_name = "GRLS" # Placeholder or fetch if necessary
+    session.headers.update({
+        "X-Restli-Protocol-Version": "2.0.0",
+        "LinkedIn-Version": "202402" 
+    })
 
     posts_url = f"{API_REST_BASE}/posts?author={org_urn}&q=author&count={count}&sortBy=LAST_MODIFIED"
     logging.info(f"Fetching posts from URL: {posts_url}")
@@ -37,19 +92,19 @@ def fetch_linkedin_posts_core(comm_client_id, community_token, org_urn, count):
         logging.info(f"Fetched {len(raw_posts_api)} raw posts from API.")
     except requests.exceptions.RequestException as e:
         status = getattr(e.response, 'status_code', 'N/A')
-        logging.error(f"Failed to fetch posts (Status: {status}): {e}")
+        text = getattr(e.response, 'text', 'No response text')
+        logging.error(f"Failed to fetch posts (Status: {status}): {e}. Response: {text}")
         raise ValueError(f"Failed to fetch posts (Status: {status})") from e
+    except json.JSONDecodeError as e:
+        logging.error(f"Failed to decode JSON from posts response: {e}. Response text: {resp.text if resp else 'No response object'}")
+        raise ValueError("Failed to decode JSON from posts response") from e
 
     if not raw_posts_api:
         logging.info("No raw posts found.")
-        return [], {}, org_name
+        return [], {}, "DefaultOrgName" 
 
-    # Filter for valid post types if necessary, e.g., shares or ugcPosts
-    # post_urns_for_stats = [p["id"] for p in raw_posts_api if ":share:" in p["id"] or ":ugcPost:" in p["id"]]
     post_urns_for_stats = [p["id"] for p in raw_posts_api if p.get("id")]
 
-
-    # Prepare texts for summarization/classification
     post_texts_for_nlp = []
     for p in raw_posts_api:
         text_content = p.get("commentary") or \
@@ -57,39 +112,57 @@ def fetch_linkedin_posts_core(comm_client_id, community_token, org_urn, count):
                        "[No text content]"
         post_texts_for_nlp.append({"text": text_content, "id": p.get("id")})
 
-    logging.info(f"Prepared {len(post_texts_for_nlp)} posts for NLP.")
-    structured_results_list = batch_summarize_and_classify(post_texts_for_nlp)
-    # Create a dictionary for easy lookup of structured results by post ID
-    structured_results_map = {res["id"]: res for res in structured_results_list if "id" in res}
+    logging.info(f"Prepared {len(post_texts_for_nlp)} posts for NLP (summarization/classification).")
+    if 'batch_summarize_and_classify' in globals():
+        structured_results_list = batch_summarize_and_classify(post_texts_for_nlp)
+    else: 
+        logging.warning("batch_summarize_and_classify not found, using fallback.")
+        structured_results_list = [{"id": p["id"], "summary": "N/A", "category": "N/A"} for p in post_texts_for_nlp]
 
+    structured_results_map = {res["id"]: res for res in structured_results_list if "id" in res}
 
-    # Fetch statistics
     stats_map = {}
     if post_urns_for_stats:
-        for i in range(0, len(post_urns_for_stats), 20): # LinkedIn API often has batch limits
-            batch_urns = post_urns_for_stats[i:i+20]
+        batch_size_stats = 20 
+        for i in range(0, len(post_urns_for_stats), batch_size_stats):
+            batch_urns = post_urns_for_stats[i:i+batch_size_stats]
             params = {'q': 'organizationalEntity', 'organizationalEntity': org_urn}
-            for idx, urn_str in enumerate(batch_urns):
-                # Determine if it's a share or ugcPost based on URN structure (simplified)
-                key_prefix = "shares" if ":share:" in urn_str else "ugcPosts"
-                params[f"{key_prefix}[{idx}]"] = urn_str
+            share_idx = 0
+            ugc_idx = 0
+            for urn_str in batch_urns:
+                if ":share:" in urn_str:
+                    params[f"shares[{share_idx}]"] = urn_str
+                    share_idx += 1
+                elif ":ugcPost:" in urn_str:
+                    params[f"ugcPosts[{ugc_idx}]"] = urn_str
+                    ugc_idx += 1
+                else:
+                    logging.warning(f"URN {urn_str} is not a recognized share or ugcPost type for stats. Skipping.")
+                    continue
             
+            if not share_idx and not ugc_idx: 
+                continue
+
             try:
-                logging.info(f"Fetching stats for batch starting with URN: {batch_urns[0]}")
+                logging.info(f"Fetching stats for batch of {len(batch_urns)} URNs starting with URN: {batch_urns[0]}")
                 stat_resp = session.get(f"{API_REST_BASE}/organizationalEntityShareStatistics", params=params)
                 stat_resp.raise_for_status()
-                for stat_element in stat_resp.json().get("elements", []):
-                    urn = stat_element.get("share") or stat_element.get("ugcPost")
-                    if urn:
-                        stats_map[urn] = stat_element.get("totalShareStatistics", {})
-                logging.info(f"Successfully fetched stats for {len(batch_urns)} URNs. Current stats_map size: {len(stats_map)}")
+                stats_data = stat_resp.json()
+                for urn_key, stat_element_values in stats_data.get("results", {}).items(): 
+                    stats_map[urn_key] = stat_element_values.get("totalShareStatistics", {})
+                
+                if stats_data.get("errors"):
+                    for urn_errored, error_detail in stats_data.get("errors", {}).items():
+                        logging.warning(f"Error fetching stats for URN {urn_errored}: {error_detail.get('message', 'Unknown error')}")
+
+                logging.info(f"Successfully processed stats response for {len(batch_urns)} URNs. Current stats_map size: {len(stats_map)}")
             except requests.exceptions.RequestException as e:
-                logging.warning(f"Failed to fetch stats for a batch: {e}. Response: {e.response.text if e.response else 'No response'}")
-                # Continue to next batch, some stats might be missing
+                status_code = getattr(e.response, 'status_code', 'N/A')
+                response_text = getattr(e.response, 'text', 'No response text')
+                logging.warning(f"Failed to fetch stats for a batch (Status: {status_code}): {e}. Params: {params}. Response: {response_text}")
             except json.JSONDecodeError as e:
                 logging.warning(f"Failed to decode JSON from stats response: {e}. Response: {stat_resp.text if stat_resp else 'No response text'}")
 
-
     processed_raw_posts = []
     for p in raw_posts_api:
         post_id = p.get("id")
@@ -101,7 +174,7 @@ def fetch_linkedin_posts_core(comm_client_id, community_token, org_urn, count):
                        p.get("specificContent", {}).get("com.linkedin.ugc.ShareContent", {}).get("shareCommentaryV2", {}).get("text", "") or \
                        "[No text content]"
         
-        timestamp = p.get("publishedAt") or p.get("createdAt")
+        timestamp = p.get("publishedAt") or p.get("createdAt") or p.get("firstPublishedAt") 
         published_at_iso = datetime.fromtimestamp(timestamp / 1000).isoformat() if timestamp else None
         
         structured_res = structured_results_map.get(post_id, {"summary": "N/A", "category": "N/A"})
@@ -113,68 +186,72 @@ def fetch_linkedin_posts_core(comm_client_id, community_token, org_urn, count):
             "category": structured_res["category"],
             "published_at_timestamp": timestamp,
             "published_at_iso": published_at_iso,
-            # These are placeholders for actual fields from LinkedIn API response. Verify field names.
-            "organization_urn": p.get("author", "urn:li:unknown"), # e.g., "urn:li:person:xxxx" or "urn:li:organization:xxxx"
-            "is_ad": 'adContext' in p
-            #"media_type": p.get("mediaCategory", "NONE") # e.g., ARTICLE, IMAGE, VIDEO, NONE
+            "organization_urn": p.get("author", f"urn:li:organization:{org_urn.split(':')[-1]}"), 
+            "is_ad": 'adContext' in p,
+            "media_category": p.get("content",{}).get("com.linkedin.voyager.feed.render.LinkedInVideoComponent",{}).get("mediaCategory") or \
+                              p.get("content",{}).get("com.linkedin.voyager.feed.render.ImageComponent",{}).get("mediaCategory") or \
+                              p.get("content",{}).get("com.linkedin.voyager.feed.render.ArticleComponent",{}).get("mediaCategory") or "NONE"
         })
     logging.info(f"Processed {len(processed_raw_posts)} posts with core data.")
-    return processed_raw_posts, stats_map, org_name
+    return processed_raw_posts, stats_map, "DefaultOrgName"
 
 
-def fetch_comments(comm_client_id, token_dict, post_urns, stats_map):
+def fetch_comments(comm_client_id, community_token, post_urns, stats_map):
     """
     Fetches comments for a list of post URNs.
     Uses stats_map to potentially skip posts with 0 comments.
     """
-    from requests_oauthlib import OAuth2Session # Keep import here if OAuth2Session is specific to this
-    
-    linkedin_session = OAuth2Session(comm_client_id, token=token_dict)
-    # LinkedIn API versions can change, ensure this is up-to-date.
-    # Using a recent version like "202402" or as per current LinkedIn docs.
-    # The user had "202502", which might be a future version. Using a slightly older one for safety.
-    linkedin_session.headers.update({'LinkedIn-Version': "202502"})
+    token_dict = community_token if isinstance(community_token, dict) else {'access_token': community_token, 'token_type': 'Bearer'}
+    linkedin_session = create_session(comm_client_id, token=token_dict)
+    linkedin_session.headers.update({
+        'LinkedIn-Version': "202402", 
+        "X-Restli-Protocol-Version": "2.0.0" 
+        })
     
     all_comments_by_post = {}
     logging.info(f"Fetching comments for {len(post_urns)} posts.")
 
     for post_urn in post_urns:
-        # Optimization: if stats show 0 comments, skip API call for this post's comments
-        if stats_map.get(post_urn, {}).get('commentCount', 0) == 0:
+        post_stats = stats_map.get(post_urn, {})
+        comment_count_from_stats = post_stats.get("commentSummary", {}).get("totalComments", post_stats.get('commentCount', 0))
+
+        if comment_count_from_stats == 0:
             logging.info(f"Skipping comment fetch for {post_urn} as commentCount is 0 in stats_map.")
             all_comments_by_post[post_urn] = []
             continue
         
         try:
-            # According to LinkedIn docs, comments are often under /socialActions/{activityUrn}/comments
-            # or /commentsV2?q=entity&entity={activityUrn}
-            # The user's URL was /socialActions/{post_urn}/comments - this seems plausible for URNs like ugcPost URNs.
-            url = f"{API_REST_BASE}/socialActions/{post_urn}/comments"
+            encoded_post_urn = quote(post_urn, safe='')
+            url = f"{API_REST_BASE}/comments?q=entity&entityUrn={encoded_post_urn}&sortOrder=CHRONOLOGICAL"
+            
             logging.debug(f"Fetching comments from URL: {url} for post URN: {post_urn}")
             response = linkedin_session.get(url)
             
             if response.status_code == 200:
                 elements = response.json().get('elements', [])
-                comments_texts = [
-                    c.get('message', {}).get('text') 
-                    for c in elements 
-                    if c.get('message') and c.get('message', {}).get('text')
-                ]
+                comments_texts = []
+                for c in elements:
+                    comment_text = c.get('message', {}).get('text')
+                    if comment_text: 
+                         comments_texts.append(comment_text)
                 all_comments_by_post[post_urn] = comments_texts
                 logging.info(f"Fetched {len(comments_texts)} comments for {post_urn}.")
-            elif response.status_code == 403: # Forbidden, often permissions or versioning
-                 logging.warning(f"Forbidden (403) to fetch comments for {post_urn}. URL: {url}. Response: {response.text}")
+            elif response.status_code == 403:
+                 logging.warning(f"Forbidden (403) to fetch comments for {post_urn}. URL: {url}. Response: {response.text}. Check permissions or API version.")
                  all_comments_by_post[post_urn] = []
-            elif response.status_code == 404: # Not found
+            elif response.status_code == 404:
                  logging.warning(f"Comments not found (404) for {post_urn}. URL: {url}. Response: {response.text}")
                  all_comments_by_post[post_urn] = []
             else:
-                logging.error(f"Error fetching comments for {post_urn}. Status: {response.status_code}. Response: {response.text}")
+                logging.error(f"Error fetching comments for {post_urn}. Status: {response.status_code}. URL: {url}. Response: {response.text}")
                 all_comments_by_post[post_urn] = []
         except requests.exceptions.RequestException as e:
             logging.error(f"RequestException fetching comments for {post_urn}: {e}")
             all_comments_by_post[post_urn] = []
-        except Exception as e: # Catch any other unexpected errors
+        except json.JSONDecodeError as e:
+            logging.error(f"JSONDecodeError fetching comments for {post_urn}. Response: {response.text if 'response' in locals() else 'N/A'}. Error: {e}")
+            all_comments_by_post[post_urn] = []
+        except Exception as e:
             logging.error(f"Unexpected error fetching comments for {post_urn}: {e}")
             all_comments_by_post[post_urn] = []
             
@@ -182,64 +259,68 @@ def fetch_comments(comm_client_id, token_dict, post_urns, stats_map):
 
 def analyze_sentiment(all_comments_data):
     """
-    Analyzes sentiment for comments grouped by post_urn.
+    Analyzes sentiment for comments grouped by post_urn using the helper function.
     all_comments_data is a dict: {post_urn: [comment_text_1, comment_text_2,...]}
-    Returns a dict: {post_urn: {"sentiment": "DominantSentiment", "percentage": X.X}}
+    Returns a dict: {post_urn: {"sentiment": "DominantOverallSentiment", "percentage": X.X, "details": {aggregated_counts}}}
     """
     results_by_post = {}
-    logging.info(f"Analyzing sentiment for comments from {len(all_comments_data)} posts.")
+    logging.info(f"Analyzing aggregated sentiment for comments from {len(all_comments_data)} posts.")
     for post_urn, comments_list in all_comments_data.items():
-        sentiment_counts = defaultdict(int)
+        aggregated_sentiment_counts = defaultdict(int)
         total_valid_comments_for_post = 0
         
         if not comments_list:
-            results_by_post[post_urn] = {"sentiment": "Neutral 😐", "percentage": 0.0, "details": sentiment_counts}
+            results_by_post[post_urn] = {"sentiment": "Neutral 😐", "percentage": 0.0, "details": dict(aggregated_sentiment_counts)}
             continue
 
         for comment_text in comments_list:
-            if not comment_text or not comment_text.strip(): # Skip empty comments
+            if not comment_text or not comment_text.strip():
                 continue
-            try:
-                # The pipeline expects a string or list of strings.
-                # Ensure comment_text is a string.
-                analysis_result = sentiment_pipeline(str(comment_text))
-                label = analysis_result[0]['label'].upper()
-                
-                if label in ['POSITIVE', 'VERY POSITIVE']:
-                    sentiment_counts['Positive 👍'] += 1
-                elif label in ['NEGATIVE', 'VERY NEGATIVE']:
-                    sentiment_counts['Negative 👎'] += 1
-                elif label == 'NEUTRAL':
-                    sentiment_counts['Neutral 😐'] += 1
-                else: # Other labels from the model
-                    sentiment_counts['Unknown'] += 1
-                total_valid_comments_for_post += 1
-            except Exception as e:
-                logging.error(f"Sentiment analysis failed for comment under {post_urn}: '{comment_text[:50]}...'. Error: {e}")
-                sentiment_counts['Error'] += 1
+            
+            # Use the helper for individual comment sentiment
+            single_comment_sentiment = _get_sentiment_from_text(comment_text)
+            
+            # Aggregate counts
+            for label, count in single_comment_sentiment["counts"].items():
+                aggregated_sentiment_counts[label] += count
+            
+            if single_comment_sentiment["label"] != "Error": # Count valid analyses
+                total_valid_comments_for_post +=1
         
+        dominant_overall_sentiment = "Neutral 😐" # Default
+        percentage = 0.0
+
         if total_valid_comments_for_post > 0:
-            dominant_sentiment = max(sentiment_counts, key=sentiment_counts.get, default='Neutral 😐')
-            percentage = round((sentiment_counts[dominant_sentiment] / total_valid_comments_for_post) * 100, 1)
-        else: # No valid comments to analyze
-            dominant_sentiment = 'Neutral 😐'
-            percentage = 0.0
-            if sentiment_counts['Error'] > 0 : # If there were only errors
-                 dominant_sentiment = 'Error'
+            # Determine dominant sentiment from aggregated_sentiment_counts
+            # Exclude 'Error' from being a dominant sentiment unless it's the only category with counts
+            valid_sentiments = {k: v for k, v in aggregated_sentiment_counts.items() if k != 'Error' and v > 0}
+            if not valid_sentiments:
+                dominant_overall_sentiment = 'Error' if aggregated_sentiment_counts['Error'] > 0 else 'Neutral 😐'
+            else:
+                # Simple max count logic for dominance
+                dominant_overall_sentiment = max(valid_sentiments, key=valid_sentiments.get)
+            
+            if dominant_overall_sentiment != 'Error':
+                percentage = round((aggregated_sentiment_counts[dominant_overall_sentiment] / total_valid_comments_for_post) * 100, 1)
+            else: # if dominant is 'Error' or only errors were found
+                percentage = 0.0
+        elif aggregated_sentiment_counts['Error'] > 0 : # No valid comments, but errors occurred
+             dominant_overall_sentiment = 'Error'
+
 
         results_by_post[post_urn] = {
-            "sentiment": dominant_sentiment, 
+            "sentiment": dominant_overall_sentiment, 
             "percentage": percentage,
-            "details": dict(sentiment_counts) # Store counts for more detailed reporting if needed
+            "details": dict(aggregated_sentiment_counts) # Store aggregated counts
         }
-        logging.debug(f"Sentiment for {post_urn}: {results_by_post[post_urn]}")
-        
+        logging.debug(f"Aggregated sentiment for post {post_urn}: {results_by_post[post_urn]}")
+            
     return results_by_post
 
 
 def compile_detailed_posts(processed_raw_posts, stats_map, sentiments_per_post):
     """
-    Combines processed raw post data with their statistics and overall sentiment.
+    Combines processed raw post data with their statistics and overall comment sentiment.
     """
     detailed_post_list = []
     logging.info(f"Compiling detailed data for {len(processed_raw_posts)} posts.")
@@ -248,23 +329,18 @@ def compile_detailed_posts(processed_raw_posts, stats_map, sentiments_per_post):
         stats = stats_map.get(post_id, {})
 
         likes = stats.get("likeCount", 0)
-        # Use 'commentSummary' from stats for comment count if available, else 'commentCount'
-        # LinkedIn sometimes has commentSummary.totalComments
-        comments_stat_count = stats.get("commentSummary", {}).get("totalComments") if "commentSummary" in stats else stats.get("commentCount", 0)
+        comments_stat_count = stats.get("commentSummary", {}).get("totalComments", stats.get("commentCount", 0))
         
         clicks = stats.get("clickCount", 0)
         shares = stats.get("shareCount", 0)
         impressions = stats.get("impressionCount", 0)
-        unique_impressions = stats.get("uniqueImpressionsCount", 0) # Ensure this field is in API response
+        unique_impressions = stats.get("uniqueImpressionsCount", stats.get("impressionCount", 0)) 
 
-        # Calculate engagement: (likes + comments + clicks + shares) / impressions
-        # Ensure impressions is not zero to avoid DivisionByZeroError
         engagement_numerator = likes + comments_stat_count + clicks + shares
-        engagement_rate = (engagement_numerator / impressions * 100) if impressions else 0.0
+        engagement_rate = (engagement_numerator / impressions * 100) if impressions and impressions > 0 else 0.0
         
-        sentiment_info = sentiments_per_post.get(post_id, {"sentiment": "Neutral 😐", "percentage": 0.0})
+        sentiment_info = sentiments_per_post.get(post_id, {"sentiment": "Neutral 😐", "percentage": 0.0, "details": {}})
         
-        # Format text for display (escaped and truncated)
         display_text = html.escape(proc_post["raw_text"][:250]).replace("\n", "<br>") + \
                        ("..." if len(proc_post["raw_text"]) > 250 else "")
         
@@ -274,25 +350,25 @@ def compile_detailed_posts(processed_raw_posts, stats_map, sentiments_per_post):
         detailed_post_list.append({
             "id": post_id,
             "when": when_formatted,
-            "text_for_display": display_text, # Shortened, escaped text
-            "raw_text": proc_post["raw_text"], # Full original text
+            "text_for_display": display_text,
+            "raw_text": proc_post["raw_text"],
             "likes": likes,
-            "comments_stat_count": comments_stat_count, # Count from post statistics
+            "comments_stat_count": comments_stat_count,
             "clicks": clicks,
             "shares": shares,
             "impressions": impressions,
             "uniqueImpressionsCount": unique_impressions,
-            "engagement": f"{engagement_rate:.2f}%", # Formatted string
-            "engagement_raw": engagement_rate, # Raw float for potential calculations
+            "engagement": f"{engagement_rate:.2f}%",
+            "engagement_raw": engagement_rate,
             "sentiment": sentiment_info["sentiment"],
             "sentiment_percent": sentiment_info["percentage"],
-            "sentiment_details": sentiment_info.get("details", {}), # Detailed counts
+            "sentiment_details": sentiment_info.get("details", {}),
             "summary": proc_post["summary"],
             "category": proc_post["category"],
             "organization_urn": proc_post["organization_urn"],
-            "is_ad": 'adContext' in proc_post,
-            #"media_type": proc_post["media_type"],
-            "published_at": proc_post["published_at_iso"] # ISO format datetime string
+            "is_ad": proc_post["is_ad"],
+            "media_category": proc_post.get("media_category", "NONE"),
+            "published_at": proc_post["published_at_iso"]
         })
     logging.info(f"Compiled {len(detailed_post_list)} detailed posts.")
     return detailed_post_list
@@ -306,48 +382,268 @@ def prepare_data_for_bubble(detailed_posts, all_actual_comments_data):
     """
     li_posts = []
     li_post_stats = []
-    li_post_comments = [] # For individual comments
-    logging.info("Preparing data for Bubble.")
-    org_urn = detailed_posts[0]["organization_urn"]
+    li_post_comments = []
+    logging.info("Preparing posts data for Bubble.")
+
+    if not detailed_posts:
+        logging.warning("No detailed posts to prepare for Bubble.")
+        return [], [], []
+
+    org_urn_default = detailed_posts[0]["organization_urn"] if detailed_posts else "urn:li:organization:UNKNOWN"
+
     for post_data in detailed_posts:
-        # Data for LI_post table in Bubble
         li_posts.append({
             "organization_urn": post_data["organization_urn"],
-            "id": post_data["id"], # Post URN
-            "is_ad": 'adContext' in post_data,
-            #"media_type": post_data["media_type"],
-            "published_at": post_data["published_at"], # ISO datetime string
-            "sentiment": post_data["sentiment"], # Overall sentiment of the post based on its comments
-            "text": post_data["raw_text"], # Storing the full raw text
-            #"summary_text": post_data["summary"],
-            "li_eb_label": post_data["category"]
-            # Add any other fields from post_data needed for LI_post table
+            "id": post_data["id"],
+            "is_ad": post_data["is_ad"],
+            "media_category": post_data.get("media_category", "NONE"),
+            "published_at": post_data["published_at"],
+            "sentiment": post_data["sentiment"], 
+            "text": post_data["raw_text"],
+            "summary_text": post_data["summary"], 
+            "li_eb_label": post_data["category"] 
         })
 
-        # Data for LI_post_stats table in Bubble
         li_post_stats.append({
             "clickCount": post_data["clicks"],
-            "commentCount": post_data["comments_stat_count"], # From post's own stats
-            "engagement": post_data["engagement"], # Formatted string e.g., "12.34%"
+            "commentCount": post_data["comments_stat_count"],
+            "engagement": post_data["engagement_raw"], 
             "impressionCount": post_data["impressions"],
             "likeCount": post_data["likes"],
             "shareCount": post_data["shares"],
             "uniqueImpressionsCount": post_data["uniqueImpressionsCount"],
-            "post_id": post_data["id"], # Foreign key to LI_post
-            "organization_urn": org_urn
+            "post_id": post_data["id"],
+            "organization_urn": post_data["organization_urn"] 
         })
 
-    # Data for LI_post_comments table in Bubble (individual comments)
-    # This iterates through the actual comments fetched, not just the count.
     for post_urn, comments_text_list in all_actual_comments_data.items():
+        current_post_org_urn = org_urn_default 
+        for p in detailed_posts: 
+            if p["id"] == post_urn:
+                current_post_org_urn = p["organization_urn"]
+                break
+
         for single_comment_text in comments_text_list:
-            if single_comment_text and single_comment_text.strip(): # Ensure comment text is not empty
+            if single_comment_text and single_comment_text.strip():
                 li_post_comments.append({
                     "comment_text": single_comment_text,
-                    "post_id": post_urn, # Foreign key to LI_post
-                    "organization_urn": org_urn
-                    # Could add sentiment per comment here if analyzed at that granularity
+                    "post_id": post_urn,
+                    "organization_urn": current_post_org_urn
                 })
     
     logging.info(f"Prepared {len(li_posts)} posts, {len(li_post_stats)} stats entries, and {len(li_post_comments)} comments for Bubble.")
-    return li_posts, li_post_stats, li_post_comments
+    return li_posts, li_post_stats, li_post_comments
+
+# --- Mentions Retrieval Functions ---
+
+def fetch_linkedin_mentions_core(comm_client_id, community_token, org_urn, count=20):
+    """
+    Fetches raw mention notifications and the details of the posts where the organization was mentioned.
+    Returns a list of processed mention data (internal structure).
+    """
+    token_dict = community_token if isinstance(community_token, dict) else {'access_token': community_token, 'token_type': 'Bearer'}
+    session = create_session(comm_client_id, token=token_dict)
+    session.headers.update({
+        "X-Restli-Protocol-Version": "2.0.0",
+        "LinkedIn-Version": "202402" 
+    })
+
+    encoded_org_urn = quote(org_urn, safe='')
+    
+    notifications_url_base = (
+        f"{API_REST_BASE}/organizationalEntityNotifications"
+        f"?q=criteria"
+        f"&actions=List(SHARE_MENTION)" 
+        f"&organizationalEntity={encoded_org_urn}"
+        f"&count={count}" 
+    )
+
+    all_notifications = []
+    start_index = 0
+    processed_mentions_internal = [] 
+    page_count = 0 
+    max_pages = 10 
+
+    while page_count < max_pages:
+        current_url = f"{notifications_url_base}&start={start_index}"
+        logging.info(f"Fetching notifications page {page_count + 1} from URL: {current_url}")
+        try:
+            resp = session.get(current_url)
+            resp.raise_for_status()
+            data = resp.json()
+            elements = data.get("elements", [])
+            
+            if not elements: 
+                logging.info(f"No more notifications found on page {page_count + 1}. Total notifications fetched: {len(all_notifications)}.")
+                break 
+            
+            all_notifications.extend(elements)
+            
+            paging = data.get("paging", {})
+            if 'start' not in paging or 'count' not in paging or len(elements) < paging.get('count', count):
+                logging.info(f"Last page of notifications fetched. Total notifications: {len(all_notifications)}.")
+                break
+
+            start_index = paging['start'] + paging['count'] 
+            page_count += 1
+
+        except requests.exceptions.RequestException as e:
+            status = getattr(e.response, 'status_code', 'N/A')
+            text = getattr(e.response, 'text', 'No response text')
+            logging.error(f"Failed to fetch notifications (Status: {status}): {e}. Response: {text}")
+            break 
+        except json.JSONDecodeError as e:
+            logging.error(f"Failed to decode JSON from notifications response: {e}. Response: {resp.text if resp else 'No resp obj'}")
+            break
+        if page_count >= max_pages:
+            logging.info(f"Reached max_pages ({max_pages}) for fetching notifications.")
+            break
+
+    if not all_notifications:
+        logging.info("No mention notifications found after fetching.")
+        return []
+
+    mention_share_urns = list(set([ 
+        n.get("generatedActivity") for n in all_notifications 
+        if n.get("action") == "SHARE_MENTION" and n.get("generatedActivity")
+    ]))
+    
+    logging.info(f"Found {len(mention_share_urns)} unique share URNs from SHARE_MENTION notifications.")
+
+    for share_urn in mention_share_urns: 
+        encoded_share_urn = quote(share_urn, safe='')
+        post_detail_url = f"{API_REST_BASE}/posts/{encoded_share_urn}" 
+        logging.info(f"Fetching details for mentioned post: {post_detail_url}")
+        try:
+            post_resp = session.get(post_detail_url)
+            post_resp.raise_for_status()
+            post_data = post_resp.json()
+
+            commentary_raw = post_data.get("commentary") 
+            if not commentary_raw and "specificContent" in post_data: 
+                 share_content = post_data.get("specificContent", {}).get("com.linkedin.ugc.ShareContent", {})
+                 commentary_raw = share_content.get("shareCommentaryV2", {}).get("text", "")
+
+            if not commentary_raw:
+                logging.warning(f"No commentary found for share URN {share_urn}. Skipping.")
+                continue
+
+            mention_text_cleaned = extract_text_from_mention_commentary(commentary_raw)
+            timestamp = post_data.get("publishedAt") or post_data.get("createdAt") or post_data.get("firstPublishedAt")
+            published_at_iso = datetime.fromtimestamp(timestamp / 1000).isoformat() if timestamp else None
+            author_urn = post_data.get("author", "urn:li:unknown") 
+
+            processed_mentions_internal.append({
+                "mention_id": f"mention_{share_urn}", 
+                "share_urn": share_urn, 
+                "mention_text_raw": commentary_raw, 
+                "mention_text_cleaned": mention_text_cleaned, 
+                "published_at_timestamp": timestamp,
+                "published_at_iso": published_at_iso, 
+                "mentioned_by_author_urn": author_urn,
+                "organization_urn_mentioned": org_urn 
+            })
+        except requests.exceptions.RequestException as e:
+            status = getattr(e.response, 'status_code', 'N/A')
+            text = getattr(e.response, 'text', 'No response text')
+            logging.warning(f"Failed to fetch post details for share URN {share_urn} (Status: {status}): {e}. Response: {text}")
+        except json.JSONDecodeError as e:
+            logging.warning(f"Failed to decode JSON for post details {share_urn}: {e}. Response: {post_resp.text if post_resp else 'No resp obj'}")
+            
+    logging.info(f"Processed {len(processed_mentions_internal)} mentions with their post details.")
+    return processed_mentions_internal
+
+
+def analyze_mentions_sentiment(processed_mentions_list):
+    """
+    Analyzes sentiment for the text of each processed mention using the helper function.
+    Input: list of processed_mention dicts (internal structure from fetch_linkedin_mentions_core).
+    Returns: a dict {mention_id: {"sentiment_label": "DominantSentiment", "percentage": 100.0, "details": {counts}}}
+    """
+    mention_sentiments_map = {}
+    logging.info(f"Analyzing individual sentiment for {len(processed_mentions_list)} mentions.")
+
+    for mention_data in processed_mentions_list:
+        mention_internal_id = mention_data["mention_id"] # Internal ID from fetch_linkedin_mentions_core
+        text_to_analyze = mention_data.get("mention_text_cleaned", "")
+        
+        sentiment_result = _get_sentiment_from_text(text_to_analyze)
+        
+        # For single text, percentage is 100% for the dominant label if not error
+        percentage = 0.0
+        if sentiment_result["label"] != "Error" and any(sentiment_result["counts"].values()):
+            percentage = 100.0
+
+        mention_sentiments_map[mention_internal_id] = {
+            "sentiment_label": sentiment_result["label"], # The dominant sentiment label
+            "percentage": percentage, 
+            "details": dict(sentiment_result["counts"]) # Raw counts for this specific mention
+        }
+        logging.debug(f"Individual sentiment for mention {mention_internal_id}: {mention_sentiments_map[mention_internal_id]}")
+            
+    return mention_sentiments_map
+
+
+def compile_detailed_mentions(processed_mentions_list, mention_sentiments_map):
+    """
+    Combines processed mention data (internal structure) with their sentiment analysis 
+    into the user-specified output format.
+    processed_mentions_list: list of dicts from fetch_linkedin_mentions_core
+    mention_sentiments_map: dict from analyze_mentions_sentiment, keyed by "mention_id" (internal)
+                           and contains "sentiment_label".
+    """
+    detailed_mentions_output = []
+    logging.info(f"Compiling detailed data for {len(processed_mentions_list)} mentions into specified format.")
+
+    for mention_core_data in processed_mentions_list:
+        mention_internal_id = mention_core_data["mention_id"] 
+        sentiment_info = mention_sentiments_map.get(mention_internal_id, {"sentiment_label": "Neutral 😐"}) 
+        
+        date_formatted = "Unknown"
+        if mention_core_data["published_at_timestamp"]:
+            try:
+                date_formatted = datetime.fromtimestamp(mention_core_data["published_at_timestamp"] / 1000).strftime("%Y-%m-%d %H:%M")
+            except TypeError: 
+                logging.warning(f"Could not format timestamp for mention_id {mention_internal_id}")
+
+        detailed_mentions_output.append({
+            "date": date_formatted, # User-specified field name
+            "id": mention_core_data["share_urn"], # User-specified field name (URN of the post with mention)
+            "mention_text": mention_core_data["mention_text_cleaned"], # User-specified field name
+            "organization_urn": mention_core_data["organization_urn_mentioned"], # User-specified field name
+            "sentiment_label": sentiment_info["sentiment_label"] # User-specified field name
+        })
+    logging.info(f"Compiled {len(detailed_mentions_output)} detailed mentions with specified fields.")
+    return detailed_mentions_output
+
+
+def prepare_mentions_for_bubble(compiled_detailed_mentions_list):
+    """
+    Prepares mention data for uploading to a Bubble table.
+    The input `compiled_detailed_mentions_list` is already in the user-specified format:
+    [{"date": ..., "id": ..., "mention_text": ..., "organization_urn": ..., "sentiment_label": ...}, ...]
+    This function directly uses these fields as per user's selection for Bubble upload.
+    """
+    li_mentions_bubble = []
+    logging.info(f"Preparing {len(compiled_detailed_mentions_list)} compiled mentions for Bubble upload.")
+
+    if not compiled_detailed_mentions_list:
+        return []
+
+    for mention_data in compiled_detailed_mentions_list:
+        # The mention_data dictionary already has the keys:
+        # "date", "id", "mention_text", "organization_urn", "sentiment_label"
+        # These are used directly for the Bubble upload list.
+        li_mentions_bubble.append({
+            "date": mention_data["date"],
+            "id": mention_data["id"], 
+            "mention_text": mention_data["mention_text"],
+            "organization_urn": mention_data["organization_urn"],
+            "sentiment_label": mention_data["sentiment_label"]
+            # If Bubble table has different field names, mapping would be done here.
+            # Example: "bubble_mention_date": mention_data["date"],
+            # For now, using direct mapping as per user's selected code for the append.
+        })
+    
+    logging.info(f"Prepared {len(li_mentions_bubble)} mention entries for Bubble, using direct field names from compiled data.")
+    return li_mentions_bubble