Update linkedin_follower_stats.py

linkedin_follower_stats.py

@@ -27,20 +27,20 @@ def _fetch_linkedin_names(session, url, params, result_key_path, name_key_path,
     """
     mapping = {}
     try:
-        logging.debug(f"Fetching names from URL: {url} with params: {params}")
+        logging.debug(f"Fetching names from URL: {url} with params: {json.dumps(params)}") # Log params for clarity
         response = session.get(url, params=params)
         response.raise_for_status()
         data = response.json()
 
         items = data
-        for key in result_key_path: # Navigate to the list/dict of items
+        for key in result_key_path: 
             if isinstance(items, dict):
                 items = items.get(key, []) 
             else: 
                 logging.warning(f"Expected dict to get key '{key}' but got {type(items)} at path {result_key_path} for URL {url}. Check result_key_path.")
                 return mapping 
 
-        if isinstance(items, dict): # For batch responses like geo/industry (where keys are IDs)
+        if isinstance(items, dict): 
             for item_id_str, item_data in items.items():
                 name = item_data
                 for key_nav in name_key_path: 
@@ -53,7 +53,7 @@ def _fetch_linkedin_names(session, url, params, result_key_path, name_key_path,
                     mapping[item_id_str] = name 
                 else:
                     logging.warning(f"No name found for ID {item_id_str} at path {name_key_path} in item: {item_data} from URL {url}")
-        elif isinstance(items, list): # For list responses like functions/seniorities
+        elif isinstance(items, list): 
             for item in items:
                 item_id_val = item.get(id_key)
                 name = item
@@ -72,7 +72,7 @@ def _fetch_linkedin_names(session, url, params, result_key_path, name_key_path,
             
     except requests.exceptions.RequestException as e:
         status_code = getattr(e.response, 'status_code', 'N/A')
-        error_text = getattr(e.response, 'text', str(e))
+        error_text = getattr(e.response, 'text', str(e)) # Log the raw error text
         logging.error(f"Error fetching names from {url} (Status: {status_code}): {error_text}")
     except json.JSONDecodeError as e:
         logging.error(f"Error decoding JSON for names from {url}: {e}. Response: {response.text if 'response' in locals() else 'N/A'}")
@@ -83,16 +83,14 @@ def _fetch_linkedin_names(session, url, params, result_key_path, name_key_path,
 def get_functions_map(session):
     """Fetches all LinkedIn functions and returns a map of {id: name}."""
     url = f"{API_V2_BASE}/functions"
-    # Rely on Accept-Language header from session for localization
-    params = {} 
+    params = {} # Relies on Accept-Language header from session
     logging.info("Fetching all LinkedIn functions.")
     return _fetch_linkedin_names(session, url, params, ["elements"], ["name", "localized", "en_US"], "id")
 
 def get_seniorities_map(session):
     """Fetches all LinkedIn seniorities and returns a map of {id: name}."""
     url = f"{API_V2_BASE}/seniorities"
-    # Rely on Accept-Language header from session for localization
-    params = {} 
+    params = {} # Relies on Accept-Language header from session
     logging.info("Fetching all LinkedIn seniorities.")
     return _fetch_linkedin_names(session, url, params, ["elements"], ["name", "localized", "en_US"], "id")
 
@@ -104,8 +102,13 @@ def get_industries_map(session, industry_urns, version="DEFAULT"):
     if not unique_ids: return {}
         
     url = f"{API_V2_BASE}/industryTaxonomyVersions/{version}/industries"
-    params = {'ids': unique_ids, 'locale': 'en_US'} # Corrected locale parameter
-    logging.info(f"Fetching names for {len(unique_ids)} unique industry IDs.")
+    # As per LinkedIn docs for BATCH_GET: ids={id1}&ids={id2}&locale.language=en&locale.country=US
+    params = {
+        'ids': unique_ids, # requests library will format this as ids=id1&ids=id2...
+        'locale.language': 'en',
+        'locale.country': 'US'
+    }
+    logging.info(f"Fetching names for {len(unique_ids)} unique industry IDs using BATCH_GET.")
     return _fetch_linkedin_names(session, url, params, ["results"], ["name", "localized", "en_US"])
 
 
@@ -116,9 +119,14 @@ def get_geo_map(session, geo_urns):
     unique_ids = list(set(filter(None, geo_ids)))
     if not unique_ids: return {}
 
-    ids_param_value = "List(" + ",".join(map(str,unique_ids)) + ")" 
-    locale_param = "en_US" # Corrected locale parameter
-    url = f"{API_V2_BASE}/geo?ids={quote(ids_param_value)}&locale={locale_param}"
+    # As per LinkedIn docs for BATCH_GET: ids=List(12345,23456)&locale=(language:en,country:US)
+    ids_param_string = "List(" + ",".join(map(str, unique_ids)) + ")"
+    locale_param_string = "(language:en,country:US)" # Must be exactly this string format
+
+    # Parameters must be passed in the URL string directly for this specific API format
+    # The `params` dict for session.get() will be empty.
+    url = f"{API_V2_BASE}/geo?ids={quote(ids_param_string)}&locale={quote(locale_param_string)}"
+    
     logging.info(f"Fetching names for {len(unique_ids)} unique geo IDs using URL: {url}")
     return _fetch_linkedin_names(session, url, {}, ["results"], ["defaultLocalizedName", "value"])
 
@@ -142,24 +150,29 @@ def _parse_urn_to_id(urn_string):
 def fetch_monthly_follower_gains(session, org_urn):
     """
     Fetches monthly follower gains for the last 12-13 months to ensure full coverage.
+    Uses parameter names as confirmed by user's working script.
     """
     results = []
     now = datetime.now(timezone.utc)
+    # Go back 13 months to ensure we capture at least 12 full previous months
+    # and have a buffer, as LinkedIn might report based on full previous months.
     thirteen_months_ago = now - relativedelta(months=13) 
     start_of_period = thirteen_months_ago.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
     start_ms = int(start_of_period.timestamp() * 1000)
     
-    url = (
-        f"{API_REST_BASE}/organizationalEntityFollowerStatistics"
-        f"?q=organizationalEntity"
-        f"&organizationalEntity={quote(org_urn)}"
-        f"&timeGranularity=MONTH"  # Corrected parameter name
-        f"&startTime={start_ms}"    # Corrected parameter name
-    )
-    logging.info(f"Fetching monthly follower gains from: {url}")
+    # Parameters as per user's working script and common LinkedIn patterns for time-bound stats
+    params = {
+        'q': 'organizationalEntity',
+        'organizationalEntity': org_urn,
+        'timeIntervals.timeGranularityType': 'MONTH',
+        'timeIntervals.timeRange.start': start_ms
+    }
+    url = f"{API_REST_BASE}/organizationalEntityFollowerStatistics"
+    
+    logging.info(f"Fetching monthly follower gains from: {url} with params: {json.dumps(params)}")
 
     try:
-        response = session.get(url)
+        response = session.get(url, params=params)
         response.raise_for_status()
         data = response.json()
 
@@ -171,7 +184,7 @@ def fetch_monthly_follower_gains(session, org_urn):
                 continue
 
             date_obj = datetime.fromtimestamp(start_timestamp_ms / 1000, tz=timezone.utc)
-            date_str = date_obj.strftime('%Y-%m-%d') 
+            date_str = date_obj.strftime('%Y-%m-%d') # First day of the month
             
             follower_gains = item.get("followerGains", {})
             organic_gain = follower_gains.get("organicFollowerGain", 0)
@@ -201,14 +214,17 @@ def fetch_follower_demographics(session, org_urn, functions_map, seniorities_map
     Fetches current follower demographics, applying Top-N for specified categories.
     """
     final_demographics_results = []
-    url = (
-        f"{API_REST_BASE}/organizationalEntityFollowerStatistics"
-        f"?q=organizationalEntity&organizationalEntity={quote(org_urn)}"
-    )
-    logging.info(f"Fetching follower demographics from: {url} for org URN {org_urn}")
+    # Parameters for the main demographics call
+    params = {
+        'q': 'organizationalEntity',
+        'organizationalEntity': org_urn
+    }
+    url = f"{API_REST_BASE}/organizationalEntityFollowerStatistics"
+    
+    logging.info(f"Fetching follower demographics from: {url} for org URN {org_urn} with params: {json.dumps(params)}")
 
     try:
-        response = session.get(url)
+        response = session.get(url, params=params)
         response.raise_for_status()
         data = response.json()
         
@@ -217,9 +233,8 @@ def fetch_follower_demographics(session, org_urn, functions_map, seniorities_map
             logging.warning(f"No elements found in follower demographics response for {org_urn}.")
             return []
         
-        stat_element = elements[0] # Data is usually in the first element
+        stat_element = elements[0] 
 
-        # Helper to convert raw API items for a single demographic type into our structured list
         def _get_entries_for_type(raw_items_list, type_name, id_map, id_field_name_in_item, org_urn_val):
             current_type_entries = []
             if not raw_items_list:
@@ -228,13 +243,11 @@ def fetch_follower_demographics(session, org_urn, functions_map, seniorities_map
 
             for item in raw_items_list:
                 category_name_val = "Unknown"
-                # For associationType, the id_field_name_in_item is the direct name
                 if type_name == "follower_association":
                     category_name_val = item.get(id_field_name_in_item, f"Unknown {id_field_name_in_item}")
-                else: # For URN-based categories
+                else: 
                     urn_val = item.get(id_field_name_in_item)
                     entity_id = _parse_urn_to_id(urn_val)
-                    # Use str(entity_id) for map lookup as map keys were stored as strings
                     category_name_val = id_map.get(str(entity_id), f"Unknown {type_name.split('_')[-1].capitalize()} (ID: {entity_id if entity_id else urn_val})")
                 
                 counts = item.get("followerCounts", {})
@@ -250,7 +263,6 @@ def fetch_follower_demographics(session, org_urn, functions_map, seniorities_map
                 })
             return current_type_entries
 
-        # Fetch live maps for industries and geo as they depend on URNs from the current API response
         industry_urns_to_map = [item.get("industry") for item in stat_element.get("followerCountsByIndustry", []) if item.get("industry")]
         geo_urns_to_map = [item.get("geo") for item in stat_element.get("followerCountsByGeoCountry", []) if item.get("geo")]
         
@@ -262,7 +274,7 @@ def fetch_follower_demographics(session, org_urn, functions_map, seniorities_map
             {"items_key": "followerCountsByFunction", "type_name": "follower_function", "id_map": functions_map, "id_field": "function", "top_n": 10},
             {"items_key": "followerCountsByIndustry", "type_name": "follower_industry", "id_map": live_industries_map, "id_field": "industry", "top_n": 10},
             {"items_key": "followerCountsByGeoCountry", "type_name": "follower_geo", "id_map": live_geo_map, "id_field": "geo", "top_n": 10},
-            {"items_key": "followerCountsByAssociationType", "type_name": "follower_association", "id_map": {}, "id_field": "associationType", "top_n": None} # Keep all associations
+            {"items_key": "followerCountsByAssociationType", "type_name": "follower_association", "id_map": {}, "id_field": "associationType", "top_n": None} 
         ]
 
         for config in demographic_configs:
@@ -270,15 +282,14 @@ def fetch_follower_demographics(session, org_urn, functions_map, seniorities_map
             processed_entries = _get_entries_for_type(raw_items, config["type_name"], config["id_map"], config["id_field"], org_urn)
             
             if config["top_n"] is not None and processed_entries:
-                # Sort by organic follower count (ensure it's numeric)
-                for entry in processed_entries: # Ensure numeric for sorting
+                for entry in processed_entries: 
                     if not isinstance(entry.get("follower_count_organic"), (int, float)):
                         entry["follower_count_organic"] = 0
                 sorted_entries = sorted(processed_entries, key=lambda x: x.get("follower_count_organic", 0), reverse=True)
                 final_demographics_results.extend(sorted_entries[:config["top_n"]])
                 logging.debug(f"Added top {config['top_n']} for {config['type_name']}. Count: {len(sorted_entries[:config['top_n']])}")
             else:
-                final_demographics_results.extend(processed_entries) # Add all if top_n is None or no entries
+                final_demographics_results.extend(processed_entries) 
                 logging.debug(f"Added all for {config['type_name']}. Count: {len(processed_entries)}")
         
         logging.info(f"Processed follower demographics for {org_urn}. Total entries from all types: {len(final_demographics_results)}")
@@ -312,7 +323,7 @@ def get_linkedin_follower_stats(comm_client_id, community_token, org_urn):
         session.headers.update({
             "X-Restli-Protocol-Version": "2.0.0", 
             "LinkedIn-Version": LINKEDIN_API_VERSION,
-            # "Accept-Language": "en_US" # Consider adding if not set by create_session and locale issues persist for v2 name lookups
+            "Accept-Language": "en_US" # Explicitly set for v2 name lookups if not default in session
         })
     except Exception as e:
         logging.error(f"Failed to create session or update headers for org {org_urn}: {e}", exc_info=True)
@@ -320,9 +331,6 @@ def get_linkedin_follower_stats(comm_client_id, community_token, org_urn):
 
     logging.info(f"Starting follower stats retrieval for org: {org_urn}")
 
-    # These maps are fetched once per call to get_linkedin_follower_stats
-    # For industries and geo, the maps will be fetched live within fetch_follower_demographics
-    # as they depend on URNs from the API response itself.
     functions_map = get_functions_map(session)
     seniorities_map = get_seniorities_map(session)
     
@@ -334,7 +342,6 @@ def get_linkedin_follower_stats(comm_client_id, community_token, org_urn):
     monthly_gains = fetch_monthly_follower_gains(session, org_urn)
     all_follower_data.extend(monthly_gains)
 
-    # Pass pre-fetched function and seniority maps. Industry and Geo maps are fetched inside.
     demographics = fetch_follower_demographics(session, org_urn, functions_map, seniorities_map)
     all_follower_data.extend(demographics)