Update patentwiz/preprocess_data.py

patentwiz/preprocess_data.py

@@ -1,6 +1,7 @@
 import os
 import requests
 import zipfile
+import streamlit as st 
 import xml.etree.ElementTree as ET
 from datetime import datetime, timedelta
 import tempfile
@@ -85,18 +86,13 @@ def download_weekly_patents(year, month, day, logging):
 def filter_rf_patents(patents, keywords=None, fields=None):
     """
     Filters patents based on keywords and specified fields, with a fallback for inconsistent field names.
-    Handles both string and dictionary-type patent representations.
-    Parameters:
-        patents (list): List of patent texts (as strings or structured data).
-        keywords (list): Keywords to filter patents.
-        fields (list): Fields to search for keywords (e.g., Title, Abstract, Claims).
-    Returns:
-        list: Filtered patents.
+    Includes Streamlit-based debugging to display progress and results in the UI.
     """
+
     if keywords is None:
         keywords = ["Radio Frequency", "Antenna", "UAV", "Wireless Charging"]  # Default keywords
     if fields is None:
-        fields = ["Title", "Abstract", "Summary", "Claims"]  # Default fields
+        fields = ["Title", "Abstract", "Summary", "Claims", "Detailed Description"]  # Default fields
 
     # Standardize field names
     FIELD_NAME_MAPPING = {
@@ -111,47 +107,52 @@ def filter_rf_patents(patents, keywords=None, fields=None):
     }
 
     filtered_patents = []
-    for patent in patents:
-        # Debugging: Print patent data type
-        print(f"Processing patent: {patent}")
 
-        # Case 1: Handle string-type patents (global search)
+    # Display first 5 patents for inspection
+    st.write("Display First 5 patents for inspection")
+    for patent in patents[:5]:
+        st.json(patent)
+
+    for patent in patents:
         if isinstance(patent, str):
+            # Global keyword search for string-type patents
             if any(keyword.lower() in patent.lower() for keyword in keywords):
+                st.write(f"Match found in string patent!")
                 filtered_patents.append(patent)
-                continue
-
-        # Case 2: Handle dictionary-type patents
         elif isinstance(patent, dict):
-            # Normalize field names in the patent dictionary
+            # Normalize field names
             normalized_patent = {}
             for field, content in patent.items():
-                normalized_field = FIELD_NAME_MAPPING.get(field, field)  # Map to standard field name
+                # Map field names to standard format
+                normalized_field = FIELD_NAME_MAPPING.get(field, field)
                 normalized_patent[normalized_field] = content
-            patent = normalized_patent
 
             # Field-specific match
             matched = False
             for field in fields:
-                field_content = patent.get(field, "")
+                field_content = normalized_patent.get(field, "")
+                st.write(f"Checking field '{field}': {field_content}") 
                 if field_content and any(keyword.lower() in field_content.lower() for keyword in keywords):
-                    filtered_patents.append(patent)
+                    st.write(f"Match found in field '{field}'")
+                    filtered_patents.append(normalized_patent)
                     matched = True
                     break
 
             # Global fallback if no fields match
             if not matched:
-                full_text = " ".join(patent.values())  # Combine all fields into one string
+                full_text = " ".join(normalized_patent.values())  # Combine all fields
                 if any(keyword.lower() in full_text.lower() for keyword in keywords):
-                    filtered_patents.append(patent)
+                    st.write(f"Match found in global search!")
+                    filtered_patents.append(normalized_patent)
         else:
-            # Handle unexpected data formats gracefully
-            print(f"Unknown patent format: {type(patent)}")
+            st.write(f"Unknown patent format: {type(patent)}")  # Handle unexpected data formats
 
+    st.write(f"Total filtered patents: {len(filtered_patents)}")
     return filtered_patents
 
 
 
+
 def extract_patents(year, month, day, logging):
     """
     This function reads a patent file in XML format, splits it into individual patents, parses each