Update app.py

app.py

@@ -1,10 +1,5 @@
 import streamlit as st
-from advertools import sitemap_to_df, word_frequency
-import pandas as pd
-import matplotlib.pyplot as plt
-
-# List of common words to filter
-common_words = set(["author", "category", "product", "authors", "categories", "products", "blog", "blogs"])
+from advertools import sitemap_to_df
 
 # Sidebar inputs for sitemaps
 sitemap_urls = [
@@ -24,36 +19,10 @@ for sitemap_url in sitemap_urls:
         except Exception as e:
             st.write(f"An error occurred for {sitemap_url}:", str(e))
 
-# Analyze the combined URLs
+# Display all URLs
 if all_urls:
-    try:
-        # Perform analysis on combined URLs
-        # Extracting relevant words from the last folder, replacing hyphens with spaces
-        slugs = [url.split("/")[-2].replace("-", " ") for url in all_urls]
-
-        # Filtering out common words
-        slugs_filtered = [' '.join([word for word in slug.split() if word.lower() not in common_words]) for slug in slugs]
-
-        # Word frequency for single words, excluding common words
-        word_freq = word_frequency(slugs_filtered, phrase_len=1)
-        st.subheader("Most-frequently used words in article titles (excluding common words)")
-        st.dataframe(word_freq.head(10))
-
-        # Word frequency for two-word phrases, excluding common words
-        word_freq_phrases = word_frequency(slugs_filtered, phrase_len=2)
-        st.subheader("Most-frequently used two-word phrases in article titles (excluding common words)")
-        st.dataframe(word_freq_phrases.head(10))
-
-        # Plotting trends
-        # Since we don't have dates associated with each URL, we won't be able to plot trends based on dates.
-        # If you have a way to associate dates with each URL, we can modify this part accordingly.
-
-        # Total number of URLs
-        st.subheader("Total Number of URLs")
-        total_urls = len(all_urls)
-        st.write(f"The total number of URLs in all sitemaps is {total_urls}.")
-
-    except Exception as e:
-        st.write("An error occurred during analysis:", str(e))
+    st.subheader("All URLs from all sitemaps:")
+    for url in all_urls:
+        st.write(url)
 else:
     st.write("Please enter at least one valid sitemap URL.")