search_funcs/clean_funcs.py

# ## Some functions to clean text

# ### Some other suggested cleaning approaches
#
# #### From here: https://shravan-kuchkula.github.io/topic-modeling/#interactive-plot-showing-results-of-k-means-clustering-lda-topic-modeling-and-sentiment-analysis
#
# - remove_hyphens
# - tokenize_text
# - remove_special_characters
# - convert to lower case
# - remove stopwords
# - lemmatize the token
# - remove short tokens
# - keep only words in wordnet
# - I ADDED ON - creating custom stopwords list

# +
# Create a custom stop words list
import nltk
import re
import string
from nltk.stem import WordNetLemmatizer
from nltk.stem import PorterStemmer
from nltk.corpus import wordnet as wn
from nltk import word_tokenize

# Add calendar months onto stop words
import calendar
from tqdm import tqdm
import gradio as gr

stemmer = PorterStemmer()


nltk.download('stopwords')
nltk.download('wordnet')

#nltk.download('words')
#nltk.download('names')

#nltk.corpus.words.words('en')  

#from sklearn.feature_extraction import text
# Adding common names to stopwords

all_names = [x.lower() for x in list(nltk.corpus.names.words())]

# Adding custom words to the stopwords
custom_words = []
my_stop_words = custom_words


cal_month = (list(calendar.month_name))
cal_month = [x.lower() for x in cal_month]

# Remove blanks
cal_month = [i for i in cal_month if i]
#print(cal_month)
custom_words.extend(cal_month)
    
#my_stop_words = frozenset(text.ENGLISH_STOP_WORDS.union(custom_words).union(all_names))
#custom_stopwords = my_stop_words
# -

# #### Some of my cleaning functions
'''
# +
# Remove all html elements from the text. Inspired by this: https://stackoverflow.com/questions/9662346/python-code-to-remove-html-tags-from-a-string

def remove_email_start(text):
  cleanr = re.compile('.*importance:|.*subject:')
  cleantext = re.sub(cleanr, '', text)
  return cleantext

def remove_email_end(text):
  cleanr = re.compile('kind regards.*|many thanks.*|sincerely.*')
  cleantext = re.sub(cleanr, '', text)
  return cleantext
    
def cleanhtml(text):
  cleanr = re.compile('<.*?>|&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});|\xa0')
  cleantext = re.sub(cleanr, '', text)
  return cleantext

## The above doesn't work when there is no > at the end of the string to match the initial <. Trying this: <[^>]+> but needs work: https://stackoverflow.com/questions/2013124/regex-matching-up-to-the-first-occurrence-of-a-character

# Remove all email addresses and numbers from the text

def cleanemail(text):
  cleanr = re.compile('\S*@\S*\s?|\xa0')
  cleantext = re.sub(cleanr, '', text)
  return cleantext

def cleannum(text):
  cleanr = re.compile(r'[0-9]+')
  cleantext = re.sub(cleanr, '', text)
  return cleantext

def cleanpostcode(text):
  cleanr = re.compile(r'(\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9][A-Z]{2})|((GIR ?0A{2})\b$)|(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9]{1}?)$)|(\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]?)\b$)')
  cleantext = re.sub(cleanr, '', text)
  return cleantext

def cleanwarning(text):
  cleanr = re.compile('caution: this email originated from outside of the organization. do not click links or open attachments unless you recognize the sender and know the content is safe.')
  cleantext = re.sub(cleanr, '', text)
  return cleantext


# -

def initial_clean(texts):
    clean_texts = []
    for text in texts:
        text = remove_email_start(text)
        text = remove_email_end(text)
        text = cleanpostcode(text)
        text = remove_hyphens(text)
        text = cleanhtml(text)
        text = cleanemail(text)
        #text = cleannum(text)        
        clean_texts.append(text)
    return clean_texts
'''
# Pre-compiling the regular expressions for efficiency
email_start_pattern = re.compile('.*importance:|.*subject:')
email_end_pattern = re.compile('kind regards.*|many thanks.*|sincerely.*')
html_pattern = re.compile('<.*?>|&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});|\xa0')
email_pattern = re.compile('\S*@\S*\s?')
num_pattern = re.compile(r'[0-9]+')
postcode_pattern = re.compile(r'(\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9][A-Z]{2})|((GIR ?0A{2})\b$)|(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9]{1}?)$)|(\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]?)\b$)')
warning_pattern = re.compile('caution: this email originated from outside of the organization. do not click links or open attachments unless you recognize the sender and know the content is safe.')
nbsp_pattern = re.compile(r'&nbsp;')

def stem_sentence(sentence):

    words = sentence.split()
    stemmed_words = [stemmer.stem(word).lower().rstrip("'") for word in words]
    return stemmed_words

def stem_sentences(sentences, progress=gr.Progress()):
        """Stem each sentence in a list of sentences."""
        stemmed_sentences = [stem_sentence(sentence) for sentence in progress.tqdm(sentences)]
        return stemmed_sentences



def get_lemma_text(text):
    # Tokenize the input string into words
    tokens = word_tokenize(text)
    
    lemmas = []
    for word in tokens:
        if len(word) > 3:
            lemma = wn.morphy(word)
        else:
            lemma = None
        
        if lemma is None:
            lemmas.append(word)
        else:
            lemmas.append(lemma)
    return lemmas

def get_lemma_tokens(tokens):
    # Tokenize the input string into words
    
    lemmas = []
    for word in tokens:
        if len(word) > 3:
            lemma = wn.morphy(word)
        else:
            lemma = None
        
        if lemma is None:
            lemmas.append(word)
        else:
            lemmas.append(lemma)
    return lemmas

def initial_clean(texts , progress=gr.Progress()):
    clean_texts = []

    i = 1
    #progress(0, desc="Cleaning texts")
    for text in progress.tqdm(texts, desc = "Cleaning data", unit = "rows"):
        #print("Cleaning row: ", i)
        text = re.sub(email_start_pattern, '', text)
        text = re.sub(email_end_pattern, '', text)
        text = re.sub(postcode_pattern, '', text)
        text = remove_hyphens(text)  
        text = re.sub(html_pattern, '', text)
        text = re.sub(email_pattern, '', text)
        text = re.sub(nbsp_pattern, '', text)
        #text = re.sub(warning_pattern, '', text)
        #text = stem_sentence(text)
        text = get_lemma_text(text)
        text = ' '.join(text)
        # Uncomment the next line if you want to remove numbers as well
        # text = re.sub(num_pattern, '', text)        
        clean_texts.append(text)

        i += 1
    return clean_texts

# Sample execution
#sample_texts = [
#    "Hello, this is a test email. kind regards, John",
#    "<div>Email content here</div> many thanks, Jane",
#   "caution: this email originated from outside of the organization. do not click links or open attachments unless you recognize the sender and know the content is safe.",
#    "[email protected]",
#    "Address: 1234 Elm St, AB12 3CD"
#]

#initial_clean(sample_texts)


# +

all_names = [x.lower() for x in list(nltk.corpus.names.words())]

def remove_hyphens(text_text):
    return re.sub(r'(\w+)-(\w+)-?(\w)?', r'\1 \2 \3', text_text)

# tokenize text
def tokenize_text(text_text):
    TOKEN_PATTERN = r'\s+'
    regex_wt = nltk.RegexpTokenizer(pattern=TOKEN_PATTERN, gaps=True)
    word_tokens = regex_wt.tokenize(text_text)
    return word_tokens

def remove_characters_after_tokenization(tokens):
    pattern = re.compile('[{}]'.format(re.escape(string.punctuation)))
    filtered_tokens = filter(None, [pattern.sub('', token) for token in tokens])
    return filtered_tokens

def convert_to_lowercase(tokens):
    return [token.lower() for token in tokens if token.isalpha()]

def remove_stopwords(tokens, custom_stopwords):
    stopword_list = nltk.corpus.stopwords.words('english')
    stopword_list += my_stop_words
    filtered_tokens = [token for token in tokens if token not in stopword_list]
    return filtered_tokens

def remove_names(tokens):
    stopword_list = list(nltk.corpus.names.words())
    stopword_list = [x.lower() for x in stopword_list]
    filtered_tokens = [token for token in tokens if token not in stopword_list]
    return filtered_tokens



def remove_short_tokens(tokens):
    return [token for token in tokens if len(token) > 3]

def keep_only_words_in_wordnet(tokens):
    return [token for token in tokens if wn.synsets(token)]

def apply_lemmatize(tokens, wnl=WordNetLemmatizer()):

    def lem_word(word):
    
        if len(word) > 3: out_word = wnl.lemmatize(word)
        else: out_word = word

        return out_word

    return [lem_word(token) for token in tokens]


# +
### Do the cleaning

def cleanTexttexts(texts):
    clean_texts = []
    for text in texts:
        #text = remove_email_start(text)
        #text = remove_email_end(text)
        text = remove_hyphens(text)
        text = cleanhtml(text)
        text = cleanemail(text)
        text = cleanpostcode(text)
        text = cleannum(text)
        #text = cleanwarning(text)
        text_i = tokenize_text(text)
        text_i = remove_characters_after_tokenization(text_i)
        #text_i = remove_names(text_i)
        text_i = convert_to_lowercase(text_i)
        #text_i = remove_stopwords(text_i, my_stop_words)
        text_i = get_lemma(text_i)
        #text_i = remove_short_tokens(text_i)
        text_i = keep_only_words_in_wordnet(text_i)

        text_i = apply_lemmatize(text_i)
        clean_texts.append(text_i)
    return clean_texts


# -

def remove_dups_text(data_samples_ready, data_samples_clean, data_samples):
   # Identify duplicates in the data: https://stackoverflow.com/questions/44191465/efficiently-identify-duplicates-in-large-list-500-000
    # Only identifies the second duplicate

    seen = set()
    dupes = []

    for i, doi in enumerate(data_samples_ready):
        if doi not in seen:
            seen.add(doi)
        else:
            dupes.append(i) 
    #data_samples_ready[dupes[0:]]
    
    # To see a specific duplicated value you know the position of
    #matching = [s for s in data_samples_ready if data_samples_ready[83] in s]
    #matching
    
    # Remove duplicates only (keep first instance)
    #data_samples_ready = list( dict.fromkeys(data_samples_ready) ) # This way would keep one version of the duplicates
    
    ### Remove all duplicates including original instance
    
    # Identify ALL duplicates including initial values
    # https://stackoverflow.com/questions/11236006/identify-duplicate-values-in-a-list-in-python

    from collections import defaultdict
    D = defaultdict(list)
    for i,item in enumerate(data_samples_ready):
        D[item].append(i)
    D = {k:v for k,v in D.items() if len(v)>1}
    
    # https://stackoverflow.com/questions/952914/how-to-make-a-flat-list-out-of-a-list-of-lists
    L = list(D.values())
    flat_list_dups = [item for sublist in L for item in sublist]

    # https://stackoverflow.com/questions/11303225/how-to-remove-multiple-indexes-from-a-list-at-the-same-time
    for index in sorted(flat_list_dups, reverse=True):
        del data_samples_ready[index]
        del data_samples_clean[index]
        del data_samples[index]
    
    # Remove blanks
    data_samples_ready = [i for i in data_samples_ready if i]
    data_samples_clean = [i for i in data_samples_clean if i]
    data_samples = [i for i in data_samples if i]
    
    return data_samples_ready, data_samples_clean, flat_list_dups, data_samples