Update app.py

app.py

@@ -1,15 +1,17 @@
-# MetaDiscovery Agent - Phase 1: LOC API Integration and Metadata Gap Analysis
+# MetaDiscovery Agent - LOC API with Collection Selector and Search Endpoint + Enhanced Features
 
 import requests
 import pandas as pd
 import streamlit as st
 import plotly.express as px
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
 
 # Streamlit app header
 st.title("MetaDiscovery Agent for Library of Congress Collections")
 st.markdown("""
 This tool connects to the LOC API, retrieves metadata from a selected collection, and performs
-an initial analysis of metadata completeness.
+an analysis of metadata completeness, suggests enhancements, and identifies authority gaps.
 """)
 
 # Predefined LOC collections
@@ -24,9 +26,9 @@ collections = {
 st.sidebar.markdown("## Settings")
 selected = st.sidebar.selectbox("Select a collection", list(collections.keys()))
 collection_path = collections[selected]
-collection_url = f"https://www.loc.gov/collections/{collection_path}/?fo=json"
 
-# Display selected collection
+# Updated: Use LOC Search API with partof filter
+collection_url = f"https://www.loc.gov/search/?q=&fa=partof:{collection_path}&fo=json"
 st.sidebar.write(f"Selected Collection: {selected}")
 
 # Fetch data from LOC API
@@ -49,19 +51,47 @@ for record in records:
 
 # Create DataFrame
 metadata_df = pd.DataFrame(items)
-st.subheader("📦 Retrieved Metadata Sample")
-st.dataframe(metadata_df.head())
-
-# Metadata completeness analysis
-st.subheader("🧠 Metadata Completeness Analysis")
-completeness = metadata_df.notnull().mean() * 100
-completeness_df = pd.DataFrame({"Field": completeness.index, "Completeness (%)": completeness.values})
-
-# Plot completeness
-fig = px.bar(completeness_df, x="Field", y="Completeness (%)", title="Metadata Completeness by Field")
-st.plotly_chart(fig)
-
-# List records with missing values
-st.subheader("⚠️ Records with Incomplete Metadata")
-incomplete_records = metadata_df[metadata_df.isnull().any(axis=1)]
-st.dataframe(incomplete_records)
+
+if not metadata_df.empty:
+    st.subheader("📦 Retrieved Metadata Sample")
+    st.dataframe(metadata_df.head())
+
+    # Metadata completeness analysis
+    st.subheader("🧠 Metadata Completeness Analysis")
+    completeness = metadata_df.notnull().mean() * 100
+    completeness_df = pd.DataFrame({"Field": completeness.index, "Completeness (%)": completeness.values})
+
+    # Plot completeness
+    fig = px.bar(completeness_df, x="Field", y="Completeness (%)", title="Metadata Completeness by Field")
+    st.plotly_chart(fig)
+
+    # List records with missing values
+    st.subheader("⚠️ Records with Incomplete Metadata")
+    incomplete_records = metadata_df[metadata_df.isnull().any(axis=1)]
+    st.dataframe(incomplete_records)
+
+    # Suggest metadata using text similarity (basic example)
+    st.subheader("✨ Suggested Metadata Enhancements")
+    filled_descriptions = metadata_df[metadata_df['description'].notnull()]['description'].astype(str)
+    tfidf = TfidfVectorizer(stop_words='english')
+    tfidf_matrix = tfidf.fit_transform(filled_descriptions)
+    sim_matrix = cosine_similarity(tfidf_matrix)
+
+    suggestions = []
+    for idx, row in incomplete_records.iterrows():
+        if pd.isna(row['subject']) and pd.notna(row['description']):
+            # Find most similar description
+            desc_vec = tfidf.transform([str(row['description'])])
+            sims = cosine_similarity(desc_vec, tfidf_matrix).flatten()
+            top_idx = sims.argmax()
+            suggested_subject = metadata_df.iloc[top_idx]['subject']
+            suggestions.append((row['title'], suggested_subject))
+
+    if suggestions:
+        suggestions_df = pd.DataFrame(suggestions, columns=["Title", "Suggested Subject"])
+        st.dataframe(suggestions_df)
+    else:
+        st.info("No metadata enhancement suggestions available.")
+
+else:
+    st.warning("No metadata records found for this collection. Try selecting another one.")