Update app.py

app.py

@@ -1,6 +1,15 @@
+
 import streamlit as st
 import re
 import json
+import nltk
+from nltk.corpus import stopwords
+from nltk import FreqDist
+from graphviz import Digraph
+from collections import Counter
+
+nltk.download('punkt')
+nltk.download('stopwords')
 
 def remove_timestamps(text):
     return re.sub(r'\d{1,2}:\d{2}\n', '', text)
@@ -41,6 +50,40 @@ def unit_test(input_text):
     test_jsonl_list = create_jsonl_list(test_text_without_timestamps)
     st.write(test_jsonl_list)
 
+
+
+def extract_high_information_words(text, top_n=10):
+    words = nltk.word_tokenize(text)
+    words = [word.lower() for word in words if word.isalpha()]
+
+    stop_words = set(stopwords.words('english'))
+    filtered_words = [word for word in words if word not in stop_words]
+
+    freq_dist = FreqDist(filtered_words)
+    high_information_words = [word for word, _ in freq_dist.most_common(top_n)]
+
+    return high_information_words
+
+
+def create_relationship_graph(words):
+    graph = Digraph()
+
+    for index, word in enumerate(words):
+        graph.node(str(index), word)
+
+        if index > 0:
+            graph.edge(str(index - 1), str(index), label=str(index))
+
+    return graph
+
+
+def display_relationship_graph(words):
+    graph = create_relationship_graph(words)
+    st.graphviz_chart(graph)
+
+
+
+
 text_input = st.text_area("Enter text:", value="", height=300)
 text_without_timestamps = remove_timestamps(text_input)
 
@@ -127,3 +170,17 @@ learning works this is model free reinforcement learning the reinforcement learn
 
 unit_test(unit_test_text_2)
 
+
+
+
+
+# Adding new functionality to the existing code
+text_without_timestamps = remove_timestamps(unit_test_text_2)
+top_words = extract_high_information_words(text_without_timestamps, 10)
+st.markdown("**Top 10 High Information Words:**")
+st.write(top_words)
+
+st.markdown("**Relationship Graph:**")
+display_relationship_graph(top_words)
+
+