Update app.py

app.py

@@ -1,4 +1,4 @@
-# MetaDiscovery Agent - LOC API with Collection Selector and Search Endpoint + Enhanced Features
+# MetaDiscovery Agent - LOC API with Enhanced Completeness and Quality Analysis
 import requests
 import pandas as pd
 import numpy as np
@@ -8,13 +8,12 @@ from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.metrics.pairwise import cosine_similarity
 
 # Streamlit app header
-st.title("LOC MetaDiscovery Agent")
+st.title("MetaDiscovery Agent for Library of Congress Collections")
 st.markdown("""
 This tool connects to the LOC API, retrieves metadata from a selected collection, and performs
 an analysis of metadata completeness, suggests enhancements, and identifies authority gaps.
 """)
 
-# Updated collection URLs using the correct LOC API format
 # Updated collection URLs using the correct LOC API format
 collections = {
     "American Revolutionary War Maps": "american+revolutionary+war+maps",
@@ -40,12 +39,9 @@ headers = {
 
 try:
     response = requests.get(collection_url, headers=headers)
-    response.raise_for_status()  # Raise exception for 4XX/5XX responses
-
-    data = response.json()  # This line is missing
+    response.raise_for_status()
+    data = response.json()
 
-    
-    # Handle both possible response structures
     if "results" in data:
         records = data.get("results", [])
     elif "items" in data:
@@ -53,9 +49,8 @@ try:
     else:
         records = []
         st.error("Unexpected API response structure. No records found.")
-        
     st.write(f"Retrieved {len(records)} records")
-    
+
 except requests.exceptions.RequestException as e:
     st.error(f"API Connection Error: {e}")
     records = []
@@ -63,12 +58,10 @@ except ValueError:
     st.error("Failed to parse API response as JSON")
     records = []
 
-# Extract selected metadata fields with proper path traversal
+# Extract selected metadata fields
 items = []
 for record in records:
-    # Handle different possible data structures
     if isinstance(record, dict):
-        # For direct field access
         item = {
             "id": record.get("id", ""),
             "title": record.get("title", ""),
@@ -77,58 +70,58 @@ for record in records:
             "creator": record.get("creator", ""),
             "description": record.get("description", "")
         }
-        
-        # For nested field access (common in LOC API)
         if not item["title"] and "item" in record:
             item["title"] = record.get("item", {}).get("title", "")
         if not item["date"] and "item" in record:
             item["date"] = record.get("item", {}).get("date", "")
-        
         items.append(item)
 
-# Create DataFrame
 metadata_df = pd.DataFrame(items)
 
+# Utility functions for deeper metadata quality analysis
+def is_incomplete(value):
+    return pd.isna(value) or value in ["", "N/A", "null", None]
+
+def is_valid_date(value):
+    try:
+        pd.to_datetime(value)
+        return True
+    except:
+        return False
+
 if not metadata_df.empty:
     st.subheader("📦 Retrieved Metadata Sample")
     st.dataframe(metadata_df.head())
-    
-    # Metadata completeness analysis
+
+    # Metadata completeness analysis (enhanced)
     st.subheader("🧠 Metadata Completeness Analysis")
-    completeness = metadata_df.notnull().mean() * 100
+    completeness = metadata_df.applymap(lambda x: not is_incomplete(x)).mean() * 100
     completeness_df = pd.DataFrame({"Field": completeness.index, "Completeness (%)": completeness.values})
-    
-    # Plot completeness
     fig = px.bar(completeness_df, x="Field", y="Completeness (%)", title="Metadata Completeness by Field")
     st.plotly_chart(fig)
-    
-    # List records with missing values
+
+    # Identify incomplete records
+    incomplete_mask = metadata_df.applymap(is_incomplete).any(axis=1)
+    incomplete_records = metadata_df[incomplete_mask]
+
     st.subheader("⚠️ Records with Incomplete Metadata")
-    incomplete_records = metadata_df[metadata_df.isnull().any(axis=1)]
     if not incomplete_records.empty:
         st.dataframe(incomplete_records)
     else:
         st.success("All metadata fields are complete in this collection!")
-    
-    # Show exact items that need updates
+
     st.subheader("📌 Identifiers of Items Needing Metadata Updates")
     if not incomplete_records.empty:
         st.write(incomplete_records[['id', 'title']])
     else:
         st.success("All records are complete!")
-    
-    # Suggest metadata using text similarity with better error handling
+
     st.subheader("✨ Suggested Metadata Enhancements")
-    
-    # Only process if we have descriptions and enough data
     filled_descriptions = metadata_df[metadata_df['description'].notnull()]['description'].astype(str)
-    
     if len(filled_descriptions) > 1:
         try:
             tfidf = TfidfVectorizer(stop_words='english')
             tfidf_matrix = tfidf.fit_transform(filled_descriptions)
-            sim_matrix = cosine_similarity(tfidf_matrix)
-            
             suggestions = []
             for idx, row in incomplete_records.iterrows():
                 if pd.isna(row['subject']) and pd.notna(row['description']):
@@ -136,9 +129,8 @@ if not metadata_df.empty:
                     sims = cosine_similarity(desc_vec, tfidf_matrix).flatten()
                     top_idx = sims.argmax()
                     suggested_subject = metadata_df.iloc[top_idx]['subject']
-                    if pd.notna(suggested_subject) and suggested_subject:  # Only add valid suggestions
+                    if pd.notna(suggested_subject) and suggested_subject:
                         suggestions.append((row['title'], suggested_subject))
-            
             if suggestions:
                 suggestions_df = pd.DataFrame(suggestions, columns=["Title", "Suggested Subject"])
                 st.dataframe(suggestions_df)
@@ -149,4 +141,4 @@ if not metadata_df.empty:
     else:
         st.info("Not enough descriptive data to generate metadata suggestions.")
 else:
-    st.warning("No metadata records found for this collection. Try selecting another one.")
+    st.warning("No metadata records found for this collection. Try selecting another one.")