Update search_utils.py

search_utils.py

@@ -334,13 +334,9 @@ class MetadataManager:
         return None
 
 
+
     def _get_semantic_scholar_url(self, title):
-        """Search Semantic Scholar API for a paper by title and return its URL.
-        
-        It queries the Semantic Scholar API for a matching paper.
-        If the API provides a URL, that URL is returned.
-        If not, but a paperId is provided, construct the URL from the paperId.
-        """
+        """Search Semantic Scholar API for a paper by title and return its URL."""
         try:
             response = requests.get(
                 "https://api.semanticscholar.org/graph/v1/paper/search",
@@ -351,22 +347,27 @@ class MetadataManager:
                 },
                 timeout=5
             )
-            response.raise_for_status()
+            response.raise_for_status()  # This raises for 429 or other errors
             data = response.json()
             
-            # Check if we got at least one result
             if "data" in data and len(data["data"]) > 0:
                 paper = data["data"][0]
-                # Prefer the provided URL if available
                 if paper.get("url"):
                     return paper["url"]
-                # Otherwise, build a URL using the paperId if available
                 elif paper.get("paperId"):
                     return f"https://www.semanticscholar.org/paper/{paper['paperId']}"
+        except requests.exceptions.HTTPError as http_err:
+            if response.status_code == 429:
+                # logger.error(f"Rate limit exceeded for Semantic Scholar API for '{title}'. Falling back.")
+                # Optionally, add a sleep delay here for backoff
+                time.sleep(1)  # simple backoff delay; consider exponential backoff
+            else:
+                # logger.error(f"Semantic Scholar API failed for '{title}': {http_err}")
         except Exception as e:
-            logger.error(f"Semantic Scholar API failed for '{title}': {str(e)}")
+            # logger.error(f"Semantic Scholar API failed for '{title}': {e}")
         return None
 
+
     
 class SemanticSearch:
     def __init__(self):