Change some text and set truncation for summarization model

app.py

@@ -51,7 +51,7 @@ def get_summarizer_model():
 
 # Page setup
 st.set_page_config(
-    page_title="Post-processing summarization fact checker",
+    page_title="📜 Post-processing summarization fact checker 📜",
     page_icon="",
     layout="centered",
     initial_sidebar_state="auto",
@@ -68,7 +68,7 @@ def list_all_article_names() -> list:
     for file in sorted(os.listdir('./sample-articles/')):
         if file.endswith('.txt'):
             filenames.append(file.replace('.txt', ''))
-        # Append free use possibility:
+    # Append free use possibility:
     filenames.append("Provide your own input")
     return filenames
 
@@ -101,7 +101,6 @@ def fetch_dependency_specific_contents(filename: str) -> AnyStr:
 
 def fetch_dependency_svg(filename: str) -> AnyStr:
     with open(f'./dependency-images/{filename.lower()}.txt', 'r') as f:
-        # data = f.read()
         lines = [line.rstrip() for line in f]
     return lines
 
@@ -113,9 +112,6 @@ def display_summary(summary_content: str):
 
 
 def get_all_entities_per_sentence(text):
-    # load all NER models
-    # nlp = get_spacy()
-    # tagger = get_flair_tagger()
     doc = nlp(text)
 
     sentences = list(doc.sents)
@@ -128,7 +124,7 @@ def get_all_entities_per_sentence(text):
         for entity in sentence.ents:
             entities_this_sentence.append(str(entity))
 
-        # FLAIR ENTITIES
+        # FLAIR ENTITIES (CURRENTLY NOT USED)
         # sentence_entities = Sentence(str(sentence))
         # tagger.predict(sentence_entities)
         # for entity in sentence_entities.get_spans('ner'):
@@ -150,22 +146,17 @@ def get_all_entities(text):
 
 
 def get_and_compare_entities():
-    # article_content = fetch_article_contents(article_name)
     article_content = st.session_state.article_text
     all_entities_per_sentence = get_all_entities_per_sentence(article_content)
-    # st.session_state.entities_per_sentence_article = all_entities_per_sentence
     entities_article = list(itertools.chain.from_iterable(all_entities_per_sentence))
 
-    # summary_content = fetch_summary_contents(article_name)
     summary_content = st.session_state.summary_output
     all_entities_per_sentence = get_all_entities_per_sentence(summary_content)
-    # st.session_state.entities_per_sentence_summary = all_entities_per_sentence
     entities_summary = list(itertools.chain.from_iterable(all_entities_per_sentence))
 
     matched_entities = []
     unmatched_entities = []
     for entity in entities_summary:
-        # TODO: currently substring matching but probably should do embedding method or idk?
         if any(entity.lower() in substring_entity.lower() for substring_entity in entities_article):
             matched_entities.append(entity)
         elif any(
@@ -179,7 +170,6 @@ def get_and_compare_entities():
 
 
 def highlight_entities():
-    # summary_content = fetch_summary_contents(article_name)
     summary_content = st.session_state.summary_output
     markdown_start_red = "<mark class=\"entity\" style=\"background: rgb(238, 135, 135);\">"
     markdown_start_green = "<mark class=\"entity\" style=\"background: rgb(121, 236, 121);\">"
@@ -206,11 +196,9 @@ def check_dependency(article: bool):
     if article:
         text = st.session_state.article_text
         all_entities = get_all_entities_per_sentence(text)
-        # all_entities = st.session_state.entities_per_sentence_article
     else:
         text = st.session_state.summary_output
         all_entities = get_all_entities_per_sentence(text)
-        # all_entities = st.session_state.entities_per_sentence_summary
     doc = nlp(text)
     tok_l = doc.to_json()['tokens']
     test_list_dict_output = []
@@ -230,7 +218,6 @@ def check_dependency(article: bool):
                     continue
                 # ONE NEEDS TO BE ENTITY
                 if object_here in all_entities[i]:
-                    # all_deps = all_deps.join(str(sentence))
                     identifier = object_here + t['dep'] + object_target
                     test_list_dict_output.append({"dep": t['dep'], "cur_word_index": (t['id'] - sentence.start),
                                                   "target_word_index": (t['head'] - sentence.start),
@@ -261,23 +248,24 @@ def generate_abstractive_summary(text, type, min_len=120, max_len=512, **kwargs)
     if type == "top_p":
         text = summarization_model(text, min_length=min_len,
                                    max_length=max_len,
-                                   top_k=50, top_p=0.95, clean_up_tokenization_spaces=True)
+                                   top_k=50, top_p=0.95, clean_up_tokenization_spaces=True, truncation=True, **kwargs)
     elif type == "greedy":
         text = summarization_model(text, min_length=min_len,
-                                   max_length=max_len, clean_up_tokenization_spaces=True)
+                                   max_length=max_len, clean_up_tokenization_spaces=True, truncation=True, **kwargs)
     elif type == "top_k":
         text = summarization_model(text, min_length=min_len, max_length=max_len, top_k=50,
-                                   clean_up_tokenization_spaces=True)
+                                   clean_up_tokenization_spaces=True, truncation=True, **kwargs)
     elif type == "beam":
         text = summarization_model(text, min_length=min_len,
                                    max_length=max_len,
-                                   clean_up_tokenization_spaces=True, **kwargs)
+                                   clean_up_tokenization_spaces=True, truncation=True, **kwargs)
     summary = text[0]['summary_text'].replace("<n>", " ")
     return summary
 
 
+
 # Page
-st.title('Summarization fact checker')
+st.title('📜 Summarization fact checker 📜')
 
 # INTRODUCTION
 st.header("Introduction")
@@ -286,16 +274,14 @@ several different downstream NLP tasks. One such task is that of text summarizat
 is to generate concise and accurate summaries from input document(s). There are 2 types of summarization: extractive 
 and abstractive. **Extractive summarization** merely copies informative fragments from the input, 
 whereas **abstractive summarization** may generate novel words. A good abstractive summary should cover principal 
-information in the input and has to be linguistically fluent. This blogpost will focus on this more difficult task of 
+information in the input and has to be linguistically fluent. This interactive blogpost will focus on this more difficult task of 
 abstractive summary generation.""")
 
 st.markdown("""To generate summaries we will use the [PEGASUS] (https://huggingface.co/google/pegasus-cnn_dailymail) 
 model, producing abstractive summaries from large articles. These summaries often contain sentences with different 
-kinds of errors. Rather than improving the core model, we will look into possible post-processing steps to improve 
-the generated summaries. By comparing contents of the summary with the source text, we come up with a factualness 
-metric, indicating the trustworthiness of the generated summary. Throughout this blog, we will also explain the 
-results for some methods on specific examples. These text blocks will be indicated and they change according to the 
-currently selected article.""")
+kinds of errors. Rather than improving the core model, we will look into possible post-processing steps to detect errors 
+from the generated summaries. Throughout this blog, we will also explain the results for some methods on specific 
+examples. These text blocks will be indicated and they change according to the currently selected article.""")
 
 # Load all different models (cached) at start time of the hugginface space
 sentence_embedding_model = get_sentence_embedding_model()
@@ -304,10 +290,11 @@ nlp = get_spacy()
 summarization_model = get_summarizer_model()
 
 # GENERATING SUMMARIES PART
-st.header("Generating summaries")
+st.header("🪶 Generating summaries")
 st.markdown("Let’s start by selecting an article text for which we want to generate a summary, or you can provide "
-            "text yourself. Note that it’s suggested to provide a sufficiently large text, as otherwise the summary "
-            "generated from it might not be optimal, leading to suboptimal performance of the post-processing steps.")
+            "text yourself. Note that it’s suggested to provide a sufficiently large article, as otherwise the "
+            "summary generated from it might not be optimal, leading to suboptimal performance of the post-processing "
+            "steps. However, too long articles will be truncated and might miss information in the summary.")
 
 selected_article = st.selectbox('Select an article or provide your own:',
                                 list_all_article_names())
@@ -319,19 +306,18 @@ article_text = st.text_area(
 )
 
 summarize_button = st.button(label='Process article content',
-                             help="Generates summary and applies entity matching and dependency parsing for given article")
+                             help="Start interactive blogpost")
 
 if summarize_button:
     st.session_state.article_text = article_text
     st.markdown(
-        "Below you can find the generated summary for the article. Based on empirical research, we will discuss "
-        "two main methods that detect some common errors. We can then score different summaries, to indicate how "
+        "Below you can find the generated summary for the article. We will discuss two approaches that we found are "
+        "able to detect some common errors. Based on errors, one could then score different summaries, indicating how "
         "factual a summary is for a given article. The idea is that in production, you could generate a set of "
         "summaries for the same article, with different parameters (or even different models). By using "
         "post-processing error detection, we can then select the best possible summary.")
     if st.session_state.article_text:
-        with st.spinner('Generating summary...'):
-            # classify_comment(article_text, selected_model)
+        with st.spinner('Generating summary, this might take a while...'):
             if selected_article != "Provide your own input" and article_text == fetch_article_contents(
                     selected_article):
                 st.session_state.unchanged_text = True
@@ -367,17 +353,15 @@ if summarize_button:
 
         markdown_start_red = "<mark class=\"entity\" style=\"background: rgb(238, 135, 135);\">"
         markdown_start_green = "<mark class=\"entity\" style=\"background: rgb(121, 236, 121);\">"
-        st.markdown("We call this technique “entity matching” and here you can see what this looks like when we apply "
-                    "this method on the summary. Entities in the summary are marked  " + green_text + " when the entity "
-                                                                                                      "also exists in the "
-                                                                                                      "article, "
-                                                                                                      "while unmatched "
-                                                                                                      "entities are "
-                                                                                                      "marked " +
-                    red_text + ". Several of the example articles and their summaries indicate different errors we find "
-                               "by using this technique. Based on which article you choose, we provide a short "
-                               "explanation of the results below.",
-                    unsafe_allow_html=True)
+        st.markdown(
+            "We call this technique “entity matching” and here you can see what this looks like when we apply this "
+            "method on the summary. Entities in the summary are marked  " + green_text + " when the entity also "
+                                                                                         "exists in the article, "
+                                                                                         "while unmatched entities "
+                                                                                         "are marked " + red_text +
+            ". Several of the example articles and their summaries indicate different errors we find by using this "
+            "technique. Based on the current article, we provide a short explanation of the results below **(only for "
+            "example articles)**. ", unsafe_allow_html=True)
         if st.session_state.unchanged_text:
             entity_specific_text = fetch_entity_specific_contents(selected_article)
             soup = BeautifulSoup(entity_specific_text, features="html.parser")
@@ -397,21 +381,17 @@ if summarize_button:
     st.markdown("Here, “Jan” is the “poss” (possession modifier) of “wife”. If suddenly the summary would read “Jan’s "
                 "husband…”, there would be a dependency in the summary that is non-existent in the article itself (namely "
                 "“Jan” is the “poss” of “husband”). However, often new dependencies are introduced in the summary that "
-                "are still correct. “The borders of Ukraine” have a different dependency between “borders” and “Ukraine” "
+                "are still correct. “The borders of Ukraine” have a different dependency between “borders” and "
+                "“Ukraine” "
                 "than “Ukraine’s borders”, while both descriptions have the same meaning. So just matching all "
                 "dependencies between article and summary (as we did with entity matching) would not be a robust method.")
-    st.markdown(
-        "However, by empirical testing, we have found that there are certain dependencies which can be used for "
-        "such matching techniques. When unmatched, these specific dependencies are often an indication of a "
-        "wrongly constructed sentence. **Should I explain this more/better or is it enough that I explain by "
-        "example specific run throughs?**. We found 2(/3 TODO) common dependencies which, when present in the "
-        "summary but not in the article, are highly indicative of factualness errors. Furthermore, we only check "
-        "dependencies between an existing **entity** and its direct connections. Below we highlight all unmatched "
-        "dependencies that satisfy the discussed constraints. We also discuss the specific results for the "
-        "currently selected article.")
+    st.markdown("However, we have found that there are specific dependencies that, when unmatched, are often an "
+                "indication of a wrongly constructed sentence. We found 2(/3 TODO) common dependencies which, "
+                "when present in the summary but not in the article, are highly indicative of factualness errors. "
+                "Furthermore, we only check dependencies between an existing **entity** and its direct connections. "
+                "Below we highlight all unmatched dependencies that satisfy the discussed constraints. We also "
+                "discuss the specific results for the currently selected example article.")
     with st.spinner("Doing dependency parsing..."):
-        # TODO RIGHT IF FUNCTION (IF EXAMPLE AND IF INPUT UNCHANGED)
-        # if selected_article == 'article11':
         if st.session_state.unchanged_text:
             for cur_svg_image in fetch_dependency_svg(selected_article):
                 st.write(cur_svg_image, unsafe_allow_html=True)
@@ -431,15 +411,13 @@ if summarize_button:
 
     # OUTRO/CONCLUSION
     st.header("Wrapping up")
-    st.markdown(
-        "We have presented 2 methods that try to improve summaries via post-processing steps. Entity matching can "
-        "be used to solve hallucinations, while dependency comparison can be used to filter out some bad "
-        "sentences (and thus worse summaries). These methods highlight the possibilities of post-processing "
-        "AI-made summaries, but are only a basic introduction. As the methods were empirically tested they are "
-        "definitely not sufficiently robust for general use-cases. (something about that we tested also RE and "
-        "maybe other things).")
+    st.markdown("We have presented 2 methods that try to detect errors in summaries via post-processing steps. Entity "
+                "matching can be used to solve hallucinations, while dependency comparison can be used to filter out "
+                "some bad sentences (and thus worse summaries). These methods highlight the possibilities of "
+                "post-processing AI-made summaries, but are only a first introduction. As the methods were "
+                "empirically tested they are definitely not sufficiently robust for general use-cases.")
     st.markdown("####")
-    st.markdown("Below we generated 5 different kind of summaries from the article in which their ranks are estimated, "
+    st.markdown("(TODO) Below we generated 5 different kind of summaries from the article in which their ranks are estimated, "
                 "and hopefully the best summary (read: the one that a human would prefer or indicate as the best one) "
                 "will be at the top. TODO: implement this (at the end I think) and also put something in the text with "
                 "the actual parameters or something? ")