Spaces:
Running
Running
| import requests | |
| import pandas as pd | |
| import numpy as np | |
| import streamlit as st | |
| import matplotlib | |
| import plotly.express as px | |
| from sklearn.feature_extraction.text import TfidfVectorizer | |
| from sklearn.metrics.pairwise import cosine_similarity | |
| # Custom CSS | |
| st.markdown(""" | |
| <style> | |
| .main { | |
| background-color: #1A1A1A !important; /* dark */ | |
| color: #D3D3D3 !important; | |
| } | |
| } | |
| .block-container { | |
| background-color: #D3D3D3 !important; | |
| color: #cccccc !important; | |
| padding-left: 3rem !important; | |
| padding-right: 3rem !important; | |
| max-width: 900px; /* widen main feed */ | |
| margin: auto; /* center it */ | |
| } | |
| /* Headings */ | |
| h1, h2, h3, h4 { | |
| color: #eeeeee !important; /* brighter light gray for headings */ | |
| font-weight: 700 !important; /* bold */ | |
| margin-bottom: 1rem !important; | |
| } | |
| p, span, div { | |
| color: #cccccc !important; | |
| } | |
| /* Subheaders (optional) */ | |
| .stSubheader { | |
| color: #dddddd !important; | |
| font-size: 1.4rem !important; | |
| } | |
| /* Dataframes (optional tweak) */ | |
| .stDataFrame { | |
| background-color: #2e2e2e !important; | |
| border-radius: 10px; | |
| padding: 1rem; | |
| } | |
| section[data-testid="stSidebar"] > div:first-child { | |
| background-color: #808080 !important; | |
| padding: 1rem; | |
| border-radius: 0.5rem; | |
| color: #808080 !important; | |
| } | |
| .stMarkdown, .stTextInput, .stDataFrame { | |
| color: #1A1A1A!important; | |
| } | |
| img.banner { | |
| width: 100%; | |
| border-radius: 12px; | |
| margin-bottom: 1rem; | |
| } | |
| .stAlert { | |
| background-color: #f0f0f5 !important; | |
| color: #1A1A1A !important; | |
| padding: 1.25rem !important; | |
| font-size: 1rem !important; | |
| border-radius: 0.5rem !important; | |
| box-shadow: 0 2px 5px rgba(0, 0, 0, 0.05) !important; | |
| } | |
| header[data-testid="stHeader"] { | |
| background-color: #1A1A1A !important; | |
| } | |
| section[data-testid="stSidebar"] > div:first-child { | |
| background-color: #1A1A1A !important; | |
| color: #FFFFFF !important; | |
| padding: 2rem 1.5rem 1.5rem 1.5rem !important; | |
| border-radius: 12px; | |
| box-shadow: 0 4px 12px rgba(0, 0, 0, 0.08); | |
| font-size: 0.95rem; | |
| line-height: 1.5; | |
| } | |
| ; | |
| html, body, [data-testid="stApp"] { | |
| background-color: #1A1A1A !important; | |
| } | |
| .custom-table { | |
| background-color: #D3D3D3; | |
| color: #1A1A1A; | |
| font-family: monospace; | |
| padding: 1rem; | |
| border-radius: 8px; | |
| overflow-x: auto; | |
| white-space: pre; | |
| border: 1px solid #ccc; | |
| } | |
| .sidebar-stats { | |
| color: lightgray !important; | |
| font-size: 1.1rem !important; | |
| margin-top: 1.5rem; | |
| font-weight: 600; | |
| } | |
| .sidebar-contrast-block { | |
| background-color: #2b2b2b !important; | |
| padding: 1.25rem; | |
| border-radius: 10px; | |
| margin-top: 1.5rem; | |
| } | |
| section.main > div { /* widen main container */ | |
| max-width: 95%; | |
| padding-left: 3rem; | |
| padding-right: 3rem; | |
| } | |
| } | |
| </style> | |
| """, unsafe_allow_html=True) | |
| # Use an image from a URL for the banner | |
| st.image("https://cdn-uploads.huggingface.co/production/uploads/67351c643fe51cb1aa28f2e5/7ThcAOjbuM8ajrP85bGs4.jpeg", use_container_width=True) | |
| # Streamlit app header | |
| st.title("MetaDiscovery Agent for Library of Congress Collections") | |
| st.markdown(""" | |
| This tool connects to the LOC API, retrieves metadata from a selected collection, and performs | |
| an analysis of metadata completeness, suggests enhancements, and identifies authority gaps. | |
| """) | |
| # Updated collection URLs using the correct LOC API | |
| collections = { | |
| "American Revolutionary War Maps": "american+revolutionary+war+maps", | |
| "Civil War Maps": "civil+war+maps", | |
| "Women's Suffrage": "women+suffrage", | |
| "World War I Posters": "world+war+posters" | |
| } | |
| # Sidebar for selecting collection | |
| #st.sidebar.markdown("## Settings") | |
| # Create empty metadata_df variable to ensure it exists before checking | |
| metadata_df = pd.DataFrame() | |
| # Add a key to the selectbox to ensure it refreshes properly | |
| with st.sidebar: | |
| st.markdown(""" | |
| <div style=' | |
| background-color: #2b2b2b | |
| padding: 1.5rem; | |
| border-radius: 12px; | |
| margin-bottom: 1.5rem; | |
| box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1); | |
| '> | |
| """, unsafe_allow_html=True) | |
| selected = st.radio("Select a Collection", list(collections.keys()), key="collection_selector") | |
| st.markdown("</div>", unsafe_allow_html=True) | |
| search_query = collections[selected] | |
| # Define the collection URL | |
| collection_url = f"https://www.loc.gov/search/?q={search_query}&fo=json" | |
| # Create an empty placeholder for Quick Stats | |
| stats_placeholder = st.sidebar.empty() | |
| # Add a fetch button to make the action explicit | |
| fetch_data = True | |
| if fetch_data: | |
| # Display a loading spinner while fetching data | |
| with st.spinner(f"Fetching data for {selected}..."): | |
| # Fetch data from LOC API with spoofed User-Agent header | |
| headers = { | |
| "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/110.0.0.0 Safari/537.36" | |
| } | |
| try: | |
| response = requests.get(collection_url, headers=headers) | |
| response.raise_for_status() | |
| data = response.json() | |
| if "results" in data: | |
| records = data.get("results", []) | |
| elif "items" in data: | |
| records = data.get("items", []) | |
| else: | |
| records = [] | |
| st.error("Unexpected API response structure. No records found.") | |
| st.write(f"Retrieved {len(records)} records") | |
| except requests.exceptions.RequestException as e: | |
| st.error(f"API Connection Error: {e}") | |
| records = [] | |
| except ValueError: | |
| st.error("Failed to parse API response as JSON") | |
| records = [] | |
| # Extract selected metadata fields | |
| items = [] | |
| for record in records: | |
| if isinstance(record, dict): | |
| description = record.get("description", "") | |
| if isinstance(description, list): | |
| description = " ".join([str(d) for d in description]) | |
| item = { | |
| "id": record.get("id", ""), | |
| "title": record.get("title", ""), | |
| "date": record.get("date", ""), | |
| "subject": ", ".join(record.get("subject", [])) if isinstance(record.get("subject"), list) else record.get("subject", ""), | |
| "creator": record.get("creator", ""), | |
| "description": description | |
| } | |
| if not item["title"] and "item" in record: | |
| item["title"] = record.get("item", {}).get("title", "") | |
| if not item["date"] and "item" in record: | |
| item["date"] = record.get("item", {}).get("date", "") | |
| items.append(item) | |
| metadata_df = pd.DataFrame(items) | |
| # Define custom completeness check | |
| def is_incomplete(value): | |
| return pd.isna(value) or value in ["", "N/A", "null", None] | |
| if not metadata_df.empty: | |
| incomplete_mask = metadata_df.map(is_incomplete).any(axis=1) | |
| incomplete_count = incomplete_mask.sum() | |
| total_fields = metadata_df.size | |
| filled_fields = (~metadata_df.map(is_incomplete)).sum().sum() | |
| overall_percent = (filled_fields / total_fields) * 100 | |
| # Field-level completeness | |
| completeness = (~metadata_df.map(is_incomplete)).mean() * 100 | |
| completeness_df = pd.DataFrame({"Field": completeness.index, "Completeness (%)": completeness.values}) | |
| completeness_table = completeness_df.set_index("Field") | |
| # Sidebar Quick Stats (index hidden, orange theme) | |
| quick_stats = pd.DataFrame({ | |
| "Metric": ["Total Records", "Incomplete Records", "Percent Complete"], | |
| "Value": [len(metadata_df), incomplete_count, round(overall_percent, 1)] | |
| }) | |
| # Style it without index | |
| styled_quick_stats = ( | |
| quick_stats.style | |
| .hide(axis="index") | |
| .background_gradient(cmap="Oranges", subset=["Value"]) | |
| .format({"Value": "{:.1f}"}) | |
| ) | |
| # Add an expander and put the dataframe inside it | |
| with st.sidebar.expander("Quick Stats", expanded=True): | |
| st.dataframe( | |
| styled_quick_stats, | |
| use_container_width=True, | |
| hide_index=True | |
| ) | |
| # Calculate Top 10 Subjects | |
| if 'subject' in metadata_df.columns: | |
| top_subjects = ( | |
| metadata_df['subject'] | |
| .dropna() | |
| .str.split(',') | |
| .explode() | |
| .str.strip() | |
| .value_counts() | |
| .head(10) | |
| .to_frame(name="Count") | |
| ) | |
| #Most Common Subjects in Sidebar | |
| with st.sidebar.expander("Top 10 Most Common Subjects", expanded=True): | |
| st.dataframe( | |
| top_subjects.style.background_gradient(cmap="Greens").format("{:.0f}"), | |
| use_container_width=True, | |
| height=240 | |
| ) | |
| with st.sidebar.expander("Helpful Resources", expanded=False): | |
| st.markdown(""" | |
| <style> | |
| .sidebar-links a { | |
| color: lightgray !important; | |
| text-decoration: none !important; | |
| } | |
| .sidebar-links a:hover { | |
| text-decoration: underline !important; | |
| } | |
| </style> | |
| <div class="sidebar-links"> | |
| <ul style='padding-left: 1em'> | |
| <li><a href="https://www.loc.gov/apis/" target="_blank">LOC API Info</a></li> | |
| <li><a href="https://www.loc.gov/" target="_blank">Library of Congress Homepage</a></li> | |
| <li><a href="https://www.loc.gov/collections/" target="_blank">LOC Digital Collections</a></li> | |
| <li><a href="https://www.loc.gov/marc/" target="_blank">MARC Metadata Standards</a></li> | |
| <li><a href="https://labs.loc.gov/about-labs/digital-strategy/" target="_blank">LOC Digital Strategy</a></li> | |
| </ul> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| # Utility functions for deeper metadata quality analysis | |
| def is_incomplete(value): | |
| return pd.isna(value) or value in ["", "N/A", "null", None] | |
| def is_valid_date(value): | |
| try: | |
| pd.to_datetime(value) | |
| return True | |
| except: | |
| return False | |
| if not metadata_df.empty: | |
| st.subheader("Retrieved Metadata Sample") | |
| st.dataframe(metadata_df.head()) | |
| # Fill the placeholder created earlier | |
| st.subheader("Field Completeness Breakdown") | |
| st.markdown(""" | |
| <div style=' | |
| background-color: #2e2e2e; | |
| padding: 1.2rem; | |
| border-radius: 10px; | |
| margin-top: 1.5rem; | |
| color: lightgray; | |
| '> | |
| """, unsafe_allow_html=True) | |
| st.dataframe( | |
| completeness_table.style.background_gradient(cmap="Greens").format("{:.1f}%"), | |
| use_container_width=True, | |
| height=240 | |
| ) | |
| st.markdown("</div>", unsafe_allow_html=True) | |
| # Identify incomplete records | |
| incomplete_mask = metadata_df.map(is_incomplete).any(axis=1) | |
| incomplete_records = metadata_df[incomplete_mask] | |
| st.subheader("Suggested Metadata Enhancements") | |
| # Look for records with descriptions but missing subjects or other fields | |
| incomplete_with_desc = metadata_df[metadata_df['description'].notnull() & | |
| (metadata_df['subject'].isnull() | | |
| metadata_df['creator'].isnull())] | |
| # Reference data should be complete records with both subjects and descriptions | |
| reference_df = metadata_df[metadata_df['subject'].notnull() & | |
| metadata_df['description'].notnull() & | |
| metadata_df['creator'].notnull()] | |
| # Print debugging info | |
| st.write(f"Records with descriptions but missing fields: {len(incomplete_with_desc)}") | |
| st.write(f"Complete reference records: {len(reference_df)}") | |
| tfidf = TfidfVectorizer(stop_words='english', max_features=1000) | |
| if len(incomplete_with_desc) > 0 and len(reference_df) > 0: | |
| try: | |
| suggestions = [] | |
| # Fit TF-IDF on all complete descriptions | |
| tfidf_matrix = tfidf.fit_transform(reference_df['description'].fillna('').astype(str)) | |
| # For each incomplete record | |
| for idx, row in incomplete_with_desc.iterrows(): | |
| if pd.notna(row['description']): | |
| # Transform this record's description | |
| desc_vec = tfidf.transform([str(row['description'])]) | |
| # Get similarity scores to all reference records | |
| sims = cosine_similarity(desc_vec, tfidf_matrix).flatten() | |
| # Find the top 3 most similar records | |
| top_indices = sims.argsort()[-3:][::-1] | |
| # Get the most frequent subject among top matches | |
| top_subjects = reference_df.iloc[top_indices]['subject'].value_counts().index | |
| if len(top_subjects) > 0: | |
| suggested_subject = top_subjects[0] | |
| suggestions.append((row['title'], suggested_subject)) | |
| if suggestions: | |
| suggestions_df = pd.DataFrame(suggestions, columns=["Title", "Suggested Subject"]) | |
| # Apply similar styling as your other tables | |
| styled_suggestions = ( | |
| suggestions_df.style | |
| .background_gradient(cmap="Greens", subset=["Suggested Subject"]) | |
| .hide(axis="index") | |
| ) | |
| # Display as a dataframe with styling | |
| st.dataframe( | |
| styled_suggestions, | |
| use_container_width=True, | |
| hide_index=True, | |
| height=min(240, len(suggestions) * 35 + 38) | |
| ) | |
| else: | |
| empty_df = pd.DataFrame([["No metadata enhancement suggestions available."]], | |
| columns=["Message"]) | |
| styled_empty = empty_df.style.hide(axis="index") | |
| st.dataframe(styled_empty, use_container_width=True, hide_index=True) | |
| except Exception as e: | |
| st.error(f"Error generating metadata suggestions: {e}") | |
| st.error(f"Error details: {str(e)}") | |
| else: | |
| empty_df = pd.DataFrame([["Not enough descriptive data to generate metadata suggestions."]], | |
| columns=["Message"]) | |
| styled_empty = empty_df.style.hide(axis="index") | |
| st.dataframe(styled_empty, use_container_width=True, hide_index=True) |