Spaces:
Running
Running
File size: 3,955 Bytes
91c3d7f d707455 91c3d7f d707455 91c3d7f d707455 083533c d707455 083533c 21b5793 083533c d707455 21b5793 d707455 91c3d7f 21b5793 91c3d7f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 |
# MetaDiscovery Agent - LOC API with Collection Selector and Search Endpoint + Enhanced Features
import requests
import pandas as pd
import streamlit as st
import plotly.express as px
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
# Streamlit app header
st.title("MetaDiscovery Agent for Library of Congress Collections")
st.markdown("""
This tool connects to the LOC API, retrieves metadata from a selected collection, and performs
an analysis of metadata completeness, suggests enhancements, and identifies authority gaps.
""")
# Predefined LOC collections
collections = {
"American Revolutionary War Maps": "american-revolutionary-war-maps",
"Civil War Maps": "civil-war-maps",
"Women’s Suffrage": "womens-suffrage",
"World War I Posters": "world-war-i-posters"
}
# Sidebar for selecting collection
st.sidebar.markdown("## Settings")
selected = st.sidebar.selectbox("Select a collection", list(collections.keys()))
collection_path = collections[selected]
# Updated: Use LOC Search API with partof filter (URL encoding for colon)
collection_url = f"https://www.loc.gov/search/?q=&fa=partof%3A{collection_path}&fo=json"
st.sidebar.write(f"Selected Collection: {selected}")
# Fetch data from LOC API
response = requests.get(collection_url)
data = response.json()
# Parse metadata records
records = data.get("results", [])
# Extract selected metadata fields
items = []
for record in records:
items.append({
"id": record.get("id"),
"title": record.get("title"),
"date": record.get("date"),
"subject": record.get("subject"),
"creator": record.get("creator"),
"description": record.get("description")
})
# Create DataFrame
metadata_df = pd.DataFrame(items)
if not metadata_df.empty:
st.subheader("📦 Retrieved Metadata Sample")
st.dataframe(metadata_df.head())
# Metadata completeness analysis
st.subheader("🧠 Metadata Completeness Analysis")
completeness = metadata_df.notnull().mean() * 100
completeness_df = pd.DataFrame({"Field": completeness.index, "Completeness (%)": completeness.values})
# Plot completeness
fig = px.bar(completeness_df, x="Field", y="Completeness (%)", title="Metadata Completeness by Field")
st.plotly_chart(fig)
# List records with missing values
st.subheader("⚠️ Records with Incomplete Metadata")
incomplete_records = metadata_df[metadata_df.isnull().any(axis=1)]
st.dataframe(incomplete_records)
# Show exact items that need updates
st.subheader("📌 Identifiers of Items Needing Metadata Updates")
if not incomplete_records.empty:
st.write(incomplete_records[['id', 'title']])
else:
st.success("All records are complete!")
# Suggest metadata using text similarity (basic example)
st.subheader("✨ Suggested Metadata Enhancements")
filled_descriptions = metadata_df[metadata_df['description'].notnull()]['description'].astype(str)
tfidf = TfidfVectorizer(stop_words='english')
tfidf_matrix = tfidf.fit_transform(filled_descriptions)
sim_matrix = cosine_similarity(tfidf_matrix)
suggestions = []
for idx, row in incomplete_records.iterrows():
if pd.isna(row['subject']) and pd.notna(row['description']):
desc_vec = tfidf.transform([str(row['description'])])
sims = cosine_similarity(desc_vec, tfidf_matrix).flatten()
top_idx = sims.argmax()
suggested_subject = metadata_df.iloc[top_idx]['subject']
suggestions.append((row['title'], suggested_subject))
if suggestions:
suggestions_df = pd.DataFrame(suggestions, columns=["Title", "Suggested Subject"])
st.dataframe(suggestions_df)
else:
st.info("No metadata enhancement suggestions available.")
else:
st.warning("No metadata records found for this collection. Try selecting another one.")
|