app.py

import streamlit as st
import nltk
from nltk.tokenize import word_tokenize, NLTKWordTokenizer, regexp_tokenize, TweetTokenizer
from nltk.tokenize.treebank import TreebankWordDetokenizer

# Regression Tests: NLTKWordTokenizer

st.write('Regression Tests: NLTKWordTokenizer\nTokenizing some test strings.')

s1 = "On a $50,000 mortgage of 30 years at 8 percent, the monthly payment would be $366.88."
st.write(f"Tokenized: {word_tokenize(s1)}")

s2 = "\"We beat some pretty good teams to get here,\" Slocum said."
st.write(f"Tokenized: {word_tokenize(s2)}")

s3 = "Well, we couldn't have this predictable, cliche-ridden, \"Touched by an Angel\" (a show creator John Masius worked on) wanna-be if she didn't."
st.write(f"Tokenized: {word_tokenize(s3)}")

s4 = "I cannot cannot work under these conditions!"
st.write(f"Tokenized: {word_tokenize(s4)}")

s5 = "The company spent $30,000,000 last year."
st.write(f"Tokenized: {word_tokenize(s5)}")

s6 = "The company spent 40.75% of its income last year."
st.write(f"Tokenized: {word_tokenize(s6)}")

s7 = "He arrived at 3:00 pm."
st.write(f"Tokenized: {word_tokenize(s7)}")

s8 = "I bought these items: books, pencils, and pens."
st.write(f"Tokenized: {word_tokenize(s8)}")

s9 = "Though there were 150, 100 of them were old."
st.write(f"Tokenized: {word_tokenize(s9)}")

s10 = "There were 300,000, but that wasn't enough."
st.write(f"Tokenized: {word_tokenize(s10)}")

s11 = "It's more'n enough."
st.write(f"Tokenized: {word_tokenize(s11)}")

s = '''Good muffins cost $3.88\nin New (York).  Please (buy) me\ntwo of them.\n(Thanks).'''
expected = [(0, 4), (5, 12), (13, 17), (18, 19), (19, 23), (24, 26), (27, 30), (31, 32), (32, 36), (36, 37), (37, 38), (40, 46), (47, 48), (48, 51), (51, 52), (53, 55), (56, 59), (60, 62), (63, 68), (69, 70), (70, 76), (76, 77), (77, 78)]
st.write(f"Gathering the spans of the tokenized strings: {list(NLTKWordTokenizer().span_tokenize(s)) == expected}")

expected = ['Good', 'muffins', 'cost', '$', '3.88', 'in', 'New', '(', 'York', ')', '.', 'Please', '(', 'buy', ')', 'me', 'two', 'of', 'them.', '(', 'Thanks', ')', '.']
st.write(f"Gathering the spans of the tokenized strings: {[s[start:end] for start, end in NLTKWordTokenizer().span_tokenize(s)] == expected}")

sx1 = '\xabNow that I can do.\xbb'
expected = ['\xab', 'Now', 'that