Spaces:
Running
Running
| # MetaDiscovery Agent - LOC API with Collection Selector and Search Endpoint + Enhanced Features | |
| import requests | |
| import pandas as pd | |
| import streamlit as st | |
| import plotly.express as px | |
| from sklearn.feature_extraction.text import TfidfVectorizer | |
| from sklearn.metrics.pairwise import cosine_similarity | |
| # Streamlit app header | |
| st.title("MetaDiscovery Agent for Library of Congress Collections") | |
| st.markdown(""" | |
| This tool connects to the LOC API, retrieves metadata from a selected collection, and performs | |
| an analysis of metadata completeness, suggests enhancements, and identifies authority gaps. | |
| """) | |
| # Predefined LOC collections | |
| collections = { | |
| "American Revolutionary War Maps": "american-revolutionary-war-maps", | |
| "Civil War Maps": "civil-war-maps", | |
| "Women’s Suffrage": "womens-suffrage", | |
| "World War I Posters": "world-war-i-posters" | |
| } | |
| # Sidebar for selecting collection | |
| st.sidebar.markdown("## Settings") | |
| selected = st.sidebar.selectbox("Select a collection", list(collections.keys())) | |
| collection_path = collections[selected] | |
| # Updated: Use LOC Search API with partof filter (URL encoding for colon) | |
| collection_url = f"https://www.loc.gov/search/?q=&fa=partof%3A{collection_path}&fo=json" | |
| st.sidebar.write(f"Selected Collection: {selected}") | |
| # Fetch data from LOC API | |
| response = requests.get(collection_url) | |
| data = response.json() | |
| # Parse metadata records | |
| records = data.get("results", []) | |
| # Extract selected metadata fields | |
| items = [] | |
| for record in records: | |
| items.append({ | |
| "id": record.get("id"), | |
| "title": record.get("title"), | |
| "date": record.get("date"), | |
| "subject": record.get("subject"), | |
| "creator": record.get("creator"), | |
| "description": record.get("description") | |
| }) | |
| # Create DataFrame | |
| metadata_df = pd.DataFrame(items) | |
| if not metadata_df.empty: | |
| st.subheader("📦 Retrieved Metadata Sample") | |
| st.dataframe(metadata_df.head()) | |
| # Metadata completeness analysis | |
| st.subheader("🧠 Metadata Completeness Analysis") | |
| completeness = metadata_df.notnull().mean() * 100 | |
| completeness_df = pd.DataFrame({"Field": completeness.index, "Completeness (%)": completeness.values}) | |
| # Plot completeness | |
| fig = px.bar(completeness_df, x="Field", y="Completeness (%)", title="Metadata Completeness by Field") | |
| st.plotly_chart(fig) | |
| # List records with missing values | |
| st.subheader("⚠️ Records with Incomplete Metadata") | |
| incomplete_records = metadata_df[metadata_df.isnull().any(axis=1)] | |
| st.dataframe(incomplete_records) | |
| # Show exact items that need updates | |
| st.subheader("📌 Identifiers of Items Needing Metadata Updates") | |
| if not incomplete_records.empty: | |
| st.write(incomplete_records[['id', 'title']]) | |
| else: | |
| st.success("All records are complete!") | |
| # Suggest metadata using text similarity (basic example) | |
| st.subheader("✨ Suggested Metadata Enhancements") | |
| filled_descriptions = metadata_df[metadata_df['description'].notnull()]['description'].astype(str) | |
| tfidf = TfidfVectorizer(stop_words='english') | |
| tfidf_matrix = tfidf.fit_transform(filled_descriptions) | |
| sim_matrix = cosine_similarity(tfidf_matrix) | |
| suggestions = [] | |
| for idx, row in incomplete_records.iterrows(): | |
| if pd.isna(row['subject']) and pd.notna(row['description']): | |
| desc_vec = tfidf.transform([str(row['description'])]) | |
| sims = cosine_similarity(desc_vec, tfidf_matrix).flatten() | |
| top_idx = sims.argmax() | |
| suggested_subject = metadata_df.iloc[top_idx]['subject'] | |
| suggestions.append((row['title'], suggested_subject)) | |
| if suggestions: | |
| suggestions_df = pd.DataFrame(suggestions, columns=["Title", "Suggested Subject"]) | |
| st.dataframe(suggestions_df) | |
| else: | |
| st.info("No metadata enhancement suggestions available.") | |
| else: | |
| st.warning("No metadata records found for this collection. Try selecting another one.") | |