Update patentwiz/preprocess_data.py

patentwiz/preprocess_data.py

@@ -84,7 +84,7 @@ def download_weekly_patents(year, month, day, logging):
 
 def filter_rf_patents(patents, keywords=None, fields=None):
     """
-    Filters patents based on keywords and specified fields, with a global fallback.
+    Filters patents based on keywords and specified fields, with fallback for inconsistent field names.
     Parameters:
         patents (list): List of patent texts (as strings or structured data).
         keywords (list): Keywords to filter patents.
@@ -97,31 +97,46 @@ def filter_rf_patents(patents, keywords=None, fields=None):
     if fields is None:
         fields = ["Title", "Abstract", "Summary", "Claims"]  # Default fields
 
+    # Standardize field names
+    FIELD_NAME_MAPPING = {
+        "abstract": "Abstract",
+        "ABSTRACT": "Abstract",
+        "summary": "Summary",
+        "SUMMARY": "Summary",
+        "claims": "Claims",
+        "CLAIMS": "Claims",
+        "detailed description": "Detailed Description",
+        "DETAILED DESCRIPTION": "Detailed Description"
+    }
+
     filtered_patents = []
     for patent in patents:
-        # Global match (for string-type patents)
-        if isinstance(patent, str):
-            if any(keyword.lower() in patent.lower() for keyword in keywords):
-                filtered_patents.append(patent)
-                continue
+        # Debugging: Print patent data
+        print(f"Processing patent: {patent}")
 
-        # Field-specific match (for dictionary-type patents)
+        # Normalize field names in the patent dictionary
         if isinstance(patent, dict):
-            matched = False
-            for field in fields:
-                field_content = patent.get(field.lower(), "")
-                if field_content and any(keyword.lower() in field_content.lower() for keyword in keywords):
-                    filtered_patents.append(patent)
-                    matched = True
-                    break
-            # Global fallback if no fields match
-            if not matched:
-                full_text = " ".join(patent.values())  # Combine all fields into one string
-                if any(keyword.lower() in full_text.lower() for keyword in keywords):
-                    filtered_patents.append(patent)
-    return filtered_patents
-
+            normalized_patent = {}
+            for field, content in patent.items():
+                normalized_field = FIELD_NAME_MAPPING.get(field, field)  # Map to standard field name
+                normalized_patent[normalized_field] = content
+            patent = normalized_patent
+
+        # Field-specific match
+        matched = False
+        for field in fields:
+            field_content = patent.get(field, "")
+            if field_content and any(keyword.lower() in field_content.lower() for keyword in keywords):
+                filtered_patents.append(patent)
+                matched = True
+                break
 
+        # Global fallback if no fields match
+        if not matched:
+            full_text = " ".join(patent.values())  # Combine all fields into one string
+            if any(keyword.lower() in full_text.lower() for keyword in keywords):
+                filtered_patents.append(patent)
+    return filtered_patents
 
 
 def extract_patents(year, month, day, logging):