Duplicate from pleonova/multi-label-summary-text

Co-authored-by: Paula Leonova <[email protected]>

.github/workflows/file_size.yml

@@ -0,0 +1,16 @@
+name: Check file size
+on:               # or directly `on: [push]` to run the action on every push on any branch
+  pull_request:
+    branches: [main]
+
+  # to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  sync-to-hub:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check large files
+        uses: ActionsDesk/[email protected]
+        with:
+          filesizelimit: 10485760 # this is 10MB so we can sync to HF Spaces

.github/workflows/main.yml

@@ -0,0 +1,19 @@
+name: Sync to Hugging Face hub
+on:
+  push:
+    branches: [main]
+
+  # to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  sync-to-hub:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Push to hub
+        env:
+          HF_TOKEN: ${{ secrets.HF_MULTI_LABEL }}
+        run: git push https://pleonova:[email protected]/spaces/pleonova/multi-label-summary-text main

.gitignore

@@ -0,0 +1,4 @@
+__pycache__/
+.DS_Store
+.gitattributes
+.idea/

README.md

@@ -0,0 +1,22 @@
+---
+title: Multi Label Summary Text
+emoji: 📚
+colorFrom: indigo
+colorTo: gray
+sdk: streamlit
+app_file: app.py
+pinned: false
+duplicated_from: pleonova/multi-label-summary-text
+---
+
+#### Interactive version
+This app is hosted on HuggingFace spaces: https://huggingface.co/spaces/pleonova/multi-label-summary-text
+
+#### Objective
+The goal of this app is to identify multiple relevant labels for long text.
+
+#### Model
+facebook/bart-large-mnli zero-shot transfer-learning summarizer and classifier
+
+#### Approach
+Updating the head of the neural network, we can use the same pretrained bart model to first summarize our long text by first splitting out our long text into chunks of 1024 tokens and then generating a summary for each of the text chunks. Next, all the summaries are concanenated and the bart model is used classify the summarized text. Alternatively, one can also classify the whole text as is.

app.py

@@ -0,0 +1,366 @@
+
+from os import write
+import time
+import pandas as pd
+import base64
+from typing import Sequence
+import streamlit as st
+from sklearn.metrics import classification_report
+
+
+# from models import create_nest_sentences, load_summary_model, summarizer_gen, load_model, classifier_zero
+import models as md
+from utils import examples_load, example_long_text_load
+import json
+
+ex_text, ex_license, ex_labels, ex_glabels = examples_load()
+ex_long_text = example_long_text_load()
+
+
+# if __name__ == '__main__':
+###################################
+######## App Description ##########
+###################################
+st.markdown("### Long Text Summarization & Multi-Label Classification")
+st.write("This app summarizes and then classifies your long text(s) with multiple labels using [BART Large CNN](https://huggingface.co/facebook/bart-large-cnn) for the summarization task and [BART Large MNLI](https://huggingface.co/facebook/bart-large-mnli) for the multi-labels matching. The keywords are independently generated using [KeyBERT](https://github.com/MaartenGr/KeyBERT) and not used in any downstream tasks.")
+st.write("__Inputs__: User enters their own custom text(s) and labels.")
+st.write("__Outputs__: A summary of the text, likelihood match score for each label and a downloadable csv of the results. \
+    Includes additional options to generate a list of keywords and/or evaluate results against a list of ground truth labels, if available.")
+
+
+
+###################################
+########  Example Input  ##########
+###################################
+example_button = st.button(label='See Example')
+if example_button:
+    example_text = ex_long_text #ex_text
+    display_text = 'Excerpt from Frankenstein:' + example_text + '"\n\n' + "[This is an excerpt from Project Gutenberg's Frankenstein. " + ex_license + "]"
+    input_labels = ex_labels
+    input_glabels = ex_glabels
+    title_name = 'Frankenstein, Chapter 3'
+else:
+    display_text = ''
+    input_labels = ''
+    input_glabels = ''
+    title_name = 'Submitted Text'
+
+
+
+with st.form(key='my_form'):
+    ###################################
+    ########   Form: Step 1  ##########
+    ###################################
+    st.markdown("##### Step 1: Upload Text")
+    text_input = st.text_area("Input any text you want to summarize & classify here (keep in mind very long text will take a while to process):", display_text)
+
+    text_csv_expander = st.expander(label=f'Want to upload multiple texts at once? Expand to upload your text files below.', expanded=False)
+    with text_csv_expander:
+        st.markdown('##### Choose one of the options below:')
+        st.write("__Option A:__")
+        uploaded_text_files = st.file_uploader(label="Upload file(s) that end with the .txt suffix",
+                                              accept_multiple_files=True, key = 'text_uploader',
+                                              type='txt')
+        st.write("__Option B:__")
+        uploaded_csv_text_files = st.file_uploader(label='Upload a CSV file with two columns: "title" and "text"',
+                                                   accept_multiple_files=False, key = 'csv_text_uploader',
+                                                   type='csv')
+
+    if text_input == display_text and display_text != '':
+        text_input = example_text
+
+    gen_keywords = st.radio(
+        "Generate keywords from text? (independent from the input labels below)",
+        ('Yes', 'No')
+        )
+
+    gen_summary = st.radio(
+        "Generate summary from text? (recommended for label matching below, but will take longer)",
+        ('Yes', 'No')
+        )
+
+    ###################################
+    ########   Form: Step 2  ##########
+    ###################################
+    st.write('\n')
+    st.markdown("##### Step 2: Enter Labels")
+    labels = st.text_input('Enter possible topic labels, which can be either keywords and/or general themes (comma-separated):',input_labels, max_chars=2000)
+    labels = list(set([x.strip() for x in labels.strip().split(',') if len(x.strip()) > 0]))
+
+    labels_csv_expander = st.expander(label=f'Prefer to upload a list of labels instead? Click here to upload your CSV file.',expanded=False)
+    with labels_csv_expander:
+        uploaded_labels_file = st.file_uploader("Choose a CSV file with one column and no header, where each cell is a separate label",
+                                                key='labels_uploader')
+
+    ###################################
+    ########   Form: Step 3  ##########
+    ###################################
+    st.write('\n')
+    st.markdown("##### Step 3: Provide Ground Truth Labels (_Optional_)")
+    glabels = st.text_input('If available, enter ground truth topic labels to evaluate results, otherwise leave blank (comma-separated):',input_glabels, max_chars=2000)
+    glabels = list(set([x.strip() for x in glabels.strip().split(',') if len(x.strip()) > 0]))
+
+
+    glabels_csv_expander = st.expander(label=f'Have a file with labels for the text? Click here to upload your CSV file.', expanded=False)
+    with glabels_csv_expander:
+        st.markdown('##### Choose one of the options below:')
+        st.write("__Option A:__")
+        uploaded_onetext_glabels_file = st.file_uploader("Single Text: Choose a CSV file with one column and no header, where each cell is a separate label",
+                                                         key = 'onetext_glabels_uploader')
+        st.write("__Option B:__")
+        uploaded_multitext_glabels_file = st.file_uploader('Multiple Text: Choose a CSV file with two columns "title" and "label", with the cells in the title column matching the name of the files uploaded in step #1.',
+                                                           key = 'multitext_glabels_uploader')
+
+
+    # threshold_value = st.slider(
+    #      'Select a threshold cutoff for matching percentage (used for ground truth label evaluation)',
+    #      0.0, 1.0, (0.5))
+
+    submit_button = st.form_submit_button(label='Submit')
+
+st.write("_For improvments/suggestions, please file an issue here: https://github.com/pleonova/multi-label-summary-text_")
+
+
+###################################
+#######  Model Load Time  #########
+###################################
+with st.spinner('Loading pretrained models...'):
+    start = time.time()
+    summarizer = md.load_summary_model()   
+    s_time = round(time.time() - start,4)
+
+    start = time.time()
+    classifier = md.load_model()  
+    c_time = round(time.time() - start,4)
+
+    start = time.time()
+    kw_model = md.load_keyword_model()
+    k_time = round(time.time() - start,4)
+
+    st.spinner(f'Time taken to load various models: {k_time}s for KeyBERT model & {s_time}s for BART summarizer mnli model & {c_time}s for BART classifier mnli model.')
+    # st.success(None)
+
+
+if submit_button or example_button:
+    ###################################
+    ########   Load Text Data   #######
+    ###################################
+    if len(text_input) == 0 and len(uploaded_text_files) == 0 and uploaded_csv_text_files is None:
+        st.error("Enter some text to generate a summary")
+    else:
+
+        if len(text_input) != 0:
+            text_df = pd.DataFrame.from_dict({'title': [title_name], 'text': [text_input]})
+
+        # OPTION A
+        elif len(uploaded_text_files) != 0:
+            st.markdown("### Text Inputs")
+            st.write('Files concatenated into a dataframe:')
+            file_names = []
+            raw_texts = []
+            for uploaded_file in uploaded_text_files:
+                text = str(uploaded_file.read(), "utf-8")
+                raw_texts.append(text)
+                title_file_name = uploaded_file.name.replace('.txt','')
+                file_names.append(title_file_name)
+            text_df = pd.DataFrame({'title': file_names,
+                                      'text': raw_texts})
+            st.dataframe(text_df.head())
+            st.download_button(
+                label="Download data as CSV",
+                data=text_df.to_csv().encode('utf-8'),
+                file_name='title_text.csv',
+                mime='title_text/csv',
+            )
+        # OPTION B
+        elif uploaded_csv_text_files is not None:
+            text_df = pd.read_csv(uploaded_csv_text_files)
+
+        # Which input was used? If text area was used, ignore the 'title'
+        if len(text_input) != 0:
+            title_element = []
+        else:
+            title_element = ['title']
+
+
+        ###################################
+        ########   Text Chunks   ##########
+        ###################################
+        with st.spinner('Breaking up text into more reasonable chunks (transformers cannot exceed a 1024 token max)...'):
+            # For each body of text, create text chunks of a certain token size required for the transformer
+
+            text_chunks_lib = dict()
+            for i in range(0, len(text_df)):
+                nested_sentences = md.create_nest_sentences(document=text_df['text'][i], token_max_length=1024)
+
+                # For each chunk of sentences (within the token max)
+                text_chunks = []
+                for n in range(0, len(nested_sentences)):
+                    tc = " ".join(map(str, nested_sentences[n]))
+                    text_chunks.append(tc)
+                title_entry = text_df['title'][i]
+                text_chunks_lib[title_entry] = text_chunks
+
+
+    ################################
+    ########   Keywords   ##########
+    ################################
+    if gen_keywords == 'Yes':
+        st.markdown("### Top Keywords")
+        with st.spinner("Generating keywords from text..."):
+
+            kw_dict = dict()
+            text_chunk_counter = 0
+            for key in text_chunks_lib:
+                keywords_list = []
+                for text_chunk in text_chunks_lib[key]:
+                    text_chunk_counter += 1
+                    keywords_list += md.keyword_gen(kw_model, text_chunk)
+                    kw_dict[key] = dict(keywords_list)
+            # Display as a dataframe
+            kw_df0 = pd.DataFrame.from_dict(kw_dict).reset_index()
+            kw_df0.rename(columns={'index': 'keyword'}, inplace=True)
+            kw_df = pd.melt(kw_df0, id_vars=['keyword'], var_name='title', value_name='score').dropna()
+
+            kw_column_list = ['keyword', 'score']
+            kw_df = kw_df[kw_df['score'] > 0.25][title_element + kw_column_list].sort_values(title_element + ['score'], ascending=False).reset_index().drop(columns='index')
+
+            st.dataframe(kw_df)
+            st.download_button(
+                label="Download data as CSV",
+                data=kw_df.to_csv().encode('utf-8'),
+                file_name='title_keywords.csv',
+                mime='title_keywords/csv',
+            )
+
+ 
+    ###################################
+    ##########   Summarize   ##########
+    ###################################
+    if gen_summary == 'Yes':
+        st.markdown("### Summary")
+        with st.spinner(f'Generating summaries for {len(text_df)} texts consisting of a total of {text_chunk_counter} chunks (this may take a minute)...'):
+            sum_dict = dict()
+            for i, key in enumerate(text_chunks_lib):
+                with st.expander(label=f'({i+1}/{len(text_df)}) Expand to see intermediate summary generation details for: {key}', expanded=False):
+                    # for key in text_chunks_lib:
+                    summary = []
+                    for num_chunk, text_chunk in enumerate(text_chunks_lib[key]):
+                        chunk_summary = md.summarizer_gen(summarizer, sequence=text_chunk, maximum_tokens=400, minimum_tokens=100)
+                        summary.append(chunk_summary)
+
+                        st.markdown(f"###### Original Text Chunk {num_chunk+1}/{len(text_chunks)}" )
+                        st.markdown(text_chunk)
+                        st.markdown(f"###### Partial Summary {num_chunk+1}/{len(text_chunks)}")
+                        st.markdown(chunk_summary)
+
+                        # Combine all the summaries into a list and compress into one document, again
+                        final_summary = "\n\n".join(list(summary))
+                        sum_dict[key] = [final_summary]
+
+            sum_df = pd.DataFrame.from_dict(sum_dict).T.reset_index()
+            sum_df.columns = ['title', 'summary_text']
+            # TO DO: Make sure summary_text does not exceed the token length
+
+        st.dataframe(sum_df)
+        st.download_button(
+            label="Download data as CSV",
+            data=sum_df.to_csv().encode('utf-8'),
+            file_name='title_summary.csv',
+            mime='title_summary/csv',
+    )
+
+    ###################################
+    ##########   Classifier   #########
+    ###################################
+    if ((len(text_input) == 0 and uploaded_text_files is None and uploaded_csv_text_files is None)
+            or (len(labels) == 0 and uploaded_labels_file is None)):
+        st.error('Enter some text and at least one possible topic to see label predictions.')
+    else:
+        if gen_summary == 'Yes':
+            st.markdown("### Top Label Predictions on Summary vs Full Text")
+        else:
+            st.markdown("### Top Label Predictions on Full Text")
+
+        if uploaded_labels_file is not None:
+            labels_df = pd.read_csv(uploaded_labels_file, header=None)
+            label_list = labels_df.iloc[:, 0]
+        else:
+            label_list = labels
+
+        with st.spinner('Matching labels...(may take some time)'):
+            if gen_summary == 'Yes':
+                labels_sum_col_list = ['title', 'label', 'scores_from_summary']
+                labels_sum_df = pd.DataFrame(columns=labels_sum_col_list)
+
+            labels_full_col_list = ['title', 'label', 'scores_from_full_text']
+            labels_full_df = pd.DataFrame(columns=labels_full_col_list)
+
+            for i in range(0, len(text_df)):
+                if gen_summary == 'Yes':
+                    s_topics, s_scores = md.classifier_zero(classifier, sequence=sum_df['summary_text'][i], labels=label_list, multi_class=True)
+                    ls_df = pd.DataFrame({'label': s_topics, 'scores_from_summary': s_scores})
+                    ls_df['title'] = text_df['title'][i]
+                    labels_sum_df = pd.concat([labels_sum_df, ls_df[labels_sum_col_list]])
+
+                f_topics, f_scores = md.classifier_zero(classifier, sequence=text_df['text'][i], labels=label_list, multi_class=True)
+                lf_df = pd.DataFrame({'label': f_topics, 'scores_from_full_text': f_scores})
+                lf_df['title'] = text_df['title'][i]
+                labels_full_df = pd.concat([labels_full_df, lf_df[labels_full_col_list]])
+
+                with st.expander(f'({i+1}/{len(text_df)}) See intermediate label matching results for: {text_df["title"][i]}'):
+                    if gen_summary == 'Yes':
+                        st.dataframe(pd.merge(ls_df, lf_df, on=['title','label']))
+                    else:
+                        st.dataframe(lf_df)
+
+            if gen_summary == 'Yes':
+                label_match_df = pd.merge(labels_sum_df, labels_full_df, on=['title', 'label'])
+            else:
+                label_match_df = labels_full_df.copy()
+
+            ###################################
+            ####### Ground Truth Labels  ######
+            ###################################
+            if len(glabels) > 0:
+                gdata = pd.DataFrame({'label': glabels})
+                join_list = ['label']
+            elif uploaded_onetext_glabels_file is not None:
+                gdata = pd.read_csv(uploaded_onetext_glabels_file, header=None)
+                join_list = ['label']
+                gdata.columns = join_list
+            elif uploaded_multitext_glabels_file is not None:
+                gdata = pd.read_csv(uploaded_multitext_glabels_file)
+                join_list = ['title', 'label']
+                gdata.columns = join_list
+
+            if len(glabels) > 0 or uploaded_onetext_glabels_file is not None or uploaded_multitext_glabels_file is not None:
+                gdata['correct_match'] = True
+                label_match_df = pd.merge(label_match_df, gdata, how='left', on=join_list)
+                label_match_df['correct_match'].fillna(False, inplace=True)
+
+            st.dataframe(label_match_df) #.sort_values(['title', 'label'], ascending=[False, False]))
+            st.download_button(
+                label="Download data as CSV",
+                data=label_match_df.to_csv().encode('utf-8'),
+                file_name='title_label_sum_full.csv',
+                mime='title_label_sum_full/csv',
+            )
+
+            # if len(glabels) > 0:
+            #     st.markdown("### Evaluation Metrics")
+            #     with st.spinner('Evaluating output against ground truth...'):
+            #
+            #         section_header_description = ['Summary Label Performance', 'Original Full Text Label Performance']
+            #         data_headers = ['scores_from_summary', 'scores_from_full_text']
+            #         for i in range(0,2):
+            #             st.markdown(f"###### {section_header_description[i]}")
+            #             report = classification_report(y_true = data2[['is_true_label']],
+            #                 y_pred = (data2[[data_headers[i]]] >= threshold_value) * 1.0,
+            #                 output_dict=True)
+            #             df_report = pd.DataFrame(report).transpose()
+            #             st.markdown(f"Threshold set for: {threshold_value}")
+            #             st.dataframe(df_report)
+
+        st.success('All done!')
+        st.balloons()

example_long_text.txt

@@ -0,0 +1,18 @@
+
+I returned home not disappointed, for I have said that I had long considered those authors useless whom the professor reprobated; but I returned not at all the more inclined to recur to these studies in any shape. M. Krempe was a little squat man with a gruff voice and a repulsive countenance; the teacher, therefore, did not prepossess me in favour of his pursuits. In rather a too philosophical and connected a strain, perhaps, I have given an account of the conclusions I had come to concerning them in my early years. As a child I had not been content with the results promised by the modern professors of natural science. With a confusion of ideas only to be accounted for by my extreme youth and my want of a guide on such matters, I had retrod the steps of knowledge along the paths of time and exchanged the discoveries of recent inquirers for the dreams of forgotten alchemists. Besides, I had a contempt for the uses of modern natural philosophy. It was very different when the masters of the science sought immortality and power; such views, although futile, were grand; but now the scene was changed. The ambition of the inquirer seemed to limit itself to the annihilation of those visions on which my interest in science was chiefly founded. I was required to exchange chimeras of boundless grandeur for realities of little worth.
+
+Such were my reflections during the first two or three days of my residence at Ingolstadt, which were chiefly spent in becoming acquainted with the localities and the principal residents in my new abode. But as the ensuing week commenced, I thought of the information which M. Krempe had given me concerning the lectures. And although I could not consent to go and hear that little conceited fellow deliver sentences out of a pulpit, I recollected what he had said of M. Waldman, whom I had never seen, as he had hitherto been out of town.
+
+Partly from curiosity and partly from idleness, I went into the lecturing room, which M. Waldman entered shortly after. This professor was very unlike his colleague. He appeared about fifty years of age, but with an aspect expressive of the greatest benevolence; a few grey hairs covered his temples, but those at the back of his head were nearly black. His person was short but remarkably erect and his voice the sweetest I had ever heard. He began his lecture by a recapitulation of the history of chemistry and the various improvements made by different men of learning, pronouncing with fervour the names of the most distinguished discoverers. He then took a cursory view of the present state of the science and explained many of its elementary terms. After having made a few preparatory experiments, he concluded with a panegyric upon modern chemistry, the terms of which I shall never forget:
+
+“The ancient teachers of this science,” said he, “promised impossibilities and performed nothing. The modern masters promise very little; they know that metals cannot be transmuted and that the elixir of life is a chimera but these philosophers, whose hands seem only made to dabble in dirt, and their eyes to pore over the microscope or crucible, have indeed performed miracles. They penetrate into the recesses of nature and show how she works in her hiding-places. They ascend into the heavens; they have discovered how the blood circulates, and the nature of the air we breathe. They have acquired new and almost unlimited powers; they can command the thunders of heaven, mimic the earthquake, and even mock the invisible world with its own shadows.”
+
+Such were the professor’s words—rather let me say such the words of the fate—enounced to destroy me. As he went on I felt as if my soul were grappling with a palpable enemy; one by one the various keys were touched which formed the mechanism of my being; chord after chord was sounded, and soon my mind was filled with one thought, one conception, one purpose. So much has been done, exclaimed the soul of Frankenstein—more, far more, will I achieve; treading in the steps already marked, I will pioneer a new way, explore unknown powers, and unfold to the world the deepest mysteries of creation.
+
+I closed not my eyes that night. My internal being was in a state of insurrection and turmoil; I felt that order would thence arise, but I had no power to produce it. By degrees, after the morning’s dawn, sleep came. I awoke, and my yesternight’s thoughts were as a dream. There only remained a resolution to return to my ancient studies and to devote myself to a science for which I believed myself to possess a natural talent. On the same day I paid M. Waldman a visit. His manners in private were even more mild and attractive than in public, for there was a certain dignity in his mien during his lecture which in his own house was replaced by the greatest affability and kindness. I gave him pretty nearly the same account of my former pursuits as I had given to his fellow professor. He heard with attention the little narration concerning my studies and smiled at the names of Cornelius Agrippa and Paracelsus, but without the contempt that M. Krempe had exhibited. He said that “These were men to whose indefatigable zeal modern philosophers were indebted for most of the foundations of their knowledge. They had left to us, as an easier task, to give new names and arrange in connected classifications the facts which they in a great degree had been the instruments of bringing to light. The labours of men of genius, however erroneously directed, scarcely ever fail in ultimately turning to the solid advantage of mankind.” I listened to his statement, which was delivered without any presumption or affectation, and then added that his lecture had removed my prejudices against modern chemists; I expressed myself in measured terms, with the modesty and deference due from a youth to his instructor, without letting escape (inexperience in life would have made me ashamed) any of the enthusiasm which stimulated my intended labours. I requested his advice concerning the books I ought to procure.
+
+“I am happy,” said M. Waldman, “to have gained a disciple; and if your application equals your ability, I have no doubt of your success. Chemistry is that branch of natural philosophy in which the greatest improvements have been and may be made; it is on that account that I have made it my peculiar study; but at the same time, I have not neglected the other branches of science. A man would make but a very sorry chemist if he attended to that department of human knowledge alone. If your wish is to become really a man of science and not merely a petty experimentalist, I should advise you to apply to every branch of natural philosophy, including mathematics.”
+
+He then took me into his laboratory and explained to me the uses of his various machines, instructing me as to what I ought to procure and promising me the use of his own when I should have advanced far enough in the science not to derange their mechanism. He also gave me the list of books which I had requested, and I took my leave.
+
+Thus ended a day memorable to me; it decided my future destiny.

examples.json

@@ -0,0 +1,6 @@
+{
+"text": "Such were the professor’s words—rather let me say such the words of the fate—enounced to destroy me. As he went on I felt as if my soul were grappling with a palpable enemy; one by one the various keys were touched which formed the mechanism of my being; chord after chord was sounded, and soon my mind was filled with one thought, one conception, one purpose. So much has been done, exclaimed the soul of Frankenstein—more, far more, will I achieve; treading in the steps already marked, I will pioneer a new way, explore unknown powers, and unfold to the world the deepest mysteries of creation.",
+"long_text_license": "This eBook is for the use of anyone anywhere in the United States and most other parts of the world at no cost and with almost no restrictions whatsoever. You may copy it, give it away or re-use it under the terms of the Project Gutenberg License included with this eBook or online at www.gutenberg.org. If you are not located in the United States, you will have to check the laws of the country where you are located before using this eBook.",
+"labels":"Batman,Science,Sound,Light,Creation,Optics",
+"ground_labels":"Science,Sound,Light"
+}

models.py

@@ -0,0 +1,96 @@
+import torch
+from transformers import AutoTokenizer, AutoModelForSequenceClassification, pipeline
+import streamlit as st
+from keybert import KeyBERT
+import re
+
+
+# Reference: https://discuss.huggingface.co/t/summarization-on-long-documents/920/7
+def create_nest_sentences(document:str, token_max_length = 1024):
+  nested = []
+  sent = []
+  length = 0
+  tokenizer = AutoTokenizer.from_pretrained('facebook/bart-large-mnli')
+
+  for sentence in re.split(r'(?<=[^A-Z].[.?]) +(?=[A-Z])', document.replace("\n", ' ')):
+    tokens_in_sentence = tokenizer(str(sentence), truncation=False, padding=False)[0] # hugging face transformer tokenizer
+    length += len(tokens_in_sentence)
+
+    if length < token_max_length:
+      sent.append(sentence)
+    else:
+      nested.append(sent)
+      sent = [sentence]
+      length = 0
+
+  if sent:
+    nested.append(sent)
+  return nested
+
+# Reference: https://github.com/MaartenGr/KeyBERT
+@st.cache(allow_output_mutation=True)
+def load_keyword_model():
+  kw_model = KeyBERT()
+  return kw_model
+
+def keyword_gen(kw_model, sequence:str):
+  keywords = kw_model.extract_keywords(sequence, 
+    keyphrase_ngram_range=(1, 1),
+    stop_words='english', 
+    use_mmr=True, 
+    diversity=0.5,
+    top_n=10)
+  return keywords
+
+
+
+# Reference: https://huggingface.co/facebook/bart-large-mnli
+@st.cache(allow_output_mutation=True)
+def load_summary_model():
+    model_name = "facebook/bart-large-cnn"
+    summarizer = pipeline(task='summarization', model=model_name)
+    return summarizer
+
+# def load_summary_model():
+#     model_name = "facebook/bart-large-mnli"
+#     tokenizer = BartTokenizer.from_pretrained(model_name)
+#     model = BartForConditionalGeneration.from_pretrained(model_name)
+#     summarizer = pipeline(task='summarization', model=model, tokenizer=tokenizer, framework='pt')
+#     return summarizer
+
+def summarizer_gen(summarizer, sequence:str, maximum_tokens:int, minimum_tokens:int):
+	output = summarizer(sequence, 
+    num_beams=4, 
+    length_penalty=2.0,
+    max_length=maximum_tokens, 
+    min_length=minimum_tokens, 
+    do_sample=False, 
+    early_stopping = True,
+    no_repeat_ngram_size=3)
+	return output[0].get('summary_text')
+
+
+# # Reference: https://www.datatrigger.org/post/nlp_hugging_face/
+# # Custom summarization pipeline (to handle long articles)
+# def summarize(text, minimum_length_of_summary = 100):
+#     # Tokenize and truncate
+#     inputs = tokenizer_bart([text], truncation=True, max_length=1024, return_tensors='pt').to('cuda')
+#     # Generate summary 
+#     summary_ids = model_bart.generate(inputs['input_ids'], num_beams=4, min_length = minimum_length_of_summary, max_length=400, early_stopping=True)
+#     # Untokenize
+#     return([tokenizer_bart.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in summary_ids][0])
+
+
+# Reference: https://huggingface.co/spaces/team-zero-shot-nli/zero-shot-nli/blob/main/utils.py
+@st.cache(allow_output_mutation=True)
+def load_model():
+    model_name = "facebook/bart-large-mnli"
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    model = AutoModelForSequenceClassification.from_pretrained(model_name)
+    classifier = pipeline(task='zero-shot-classification', model=model, tokenizer=tokenizer, framework='pt')
+    return classifier
+
+def classifier_zero(classifier, sequence:str, labels:list, multi_class:bool):
+    outputs = classifier(sequence, labels, multi_label=multi_class)
+    return outputs['labels'], outputs['scores']
+

requirements.txt

@@ -0,0 +1,6 @@
+transformers[sentencepiece]==4.11.0
+pandas==1.4.3
+streamlit==1.10.0
+torch==1.12.0
+scikit-learn==1.1.1
+KeyBERT==0.5.1

utils.py

@@ -0,0 +1,85 @@
+import streamlit as st
+import numpy as np
+import pandas as pd
+# import plotly.express as px
+# from plotly.subplots import make_subplots
+import json
+
+# Reference: https://huggingface.co/spaces/team-zero-shot-nli/zero-shot-nli/blob/main/utils.py
+# def plot_result(top_topics, scores):
+#     top_topics = np.array(top_topics)
+#     scores = np.array(scores)
+#     scores *= 100
+#     fig = px.bar(x=np.around(scores,2), y=top_topics, orientation='h', 
+#                  labels={'x': 'Confidence Score', 'y': 'Label'},
+#                  text=scores,
+#                  range_x=(0,115),
+#                  title='Predictions',
+#                  color=np.linspace(0,1,len(scores)),
+#                  color_continuous_scale='GnBu')
+#     fig.update(layout_coloraxis_showscale=False)
+#     fig.update_traces(texttemplate='%{text:0.1f}%', textposition='outside')
+#     st.plotly_chart(fig)
+
+
+# def plot_dual_bar_chart(topics_summary, scores_summary, topics_text, scores_text):
+#     data1 = pd.DataFrame({'label': topics_summary, 'scores on summary': scores_summary})
+#     data2 = pd.DataFrame({'label': topics_text, 'scores on full text': scores_text})
+#     data = pd.merge(data1, data2, on = ['label'])
+#     data.sort_values('scores on summary', ascending = True, inplace = True)
+
+#     fig = make_subplots(rows=1, cols=2, 
+#         subplot_titles=("Predictions on Summary", "Predictions on Full Text"),
+#         )
+
+#     fig1 = px.bar(x=round(data['scores on summary']*100, 2), y=data['label'], orientation='h', 
+#                  text=round(data['scores on summary']*100, 2),
+#                  )
+
+#     fig2 = px.bar(x=round(data['scores on full text']*100,2), y=data['label'], orientation='h', 
+#                  text=round(data['scores on full text']*100,2),
+#                  )
+
+#     fig.add_trace(fig1['data'][0], row=1, col=1)
+#     fig.add_trace(fig2['data'][0], row=1, col=2)
+
+#     fig.update_traces(texttemplate='%{text:0.1f}%', textposition='outside')
+#     fig.update_layout(height=600, width=700) #, title_text="Predictions for")
+#     fig.update_xaxes(range=[0,115])
+#     fig.update_xaxes(matches='x')
+#     fig.update_yaxes(showticklabels=False) # hide all the xticks
+#     fig.update_yaxes(showticklabels=True, row=1, col=1)
+
+#     st.plotly_chart(fig)
+
+# def plot_dual_bar_chart(topics_summary, scores_summary, topics_text, scores_text):
+#     data1 = pd.DataFrame({'label': topics_summary, 'scores': scores_summary})
+#     data1['classification_on'] = 'summary'
+#     data2 = pd.DataFrame({'label': topics_text, 'scores': scores_text})
+#     data2['classification_on'] = 'full text'
+#     data = pd.concat([data1, data2])
+#     data['scores'] = round(data['scores']*100,2)
+
+#     fig = px.bar(
+#         data, x="scores", y="label", #orientation = 'h',
+#                  labels={'x': 'Confidence Score', 'y': 'Label'},
+#                  text=data['scores'],
+#                  range_x=(0,115),
+#                  color="label", barmode="group", 
+#                  facet_col="classification_on",
+#                  category_orders={"classification_on": ["summary", "full text"]}
+#        )
+#     fig.update_traces(texttemplate='%{text:0.1f}%', textposition='outside')
+
+#     st.plotly_chart(fig)
+
+
+def examples_load():
+    with open("examples.json") as f:
+        data=json.load(f)
+    return data['text'], data['long_text_license'], data['labels'], data['ground_labels']
+
+def example_long_text_load():
+    with open("example_long_text.txt", "r") as f:
+        text_data = f.read()
+    return text_data