nlutils from app.py

app.py

@@ -2,196 +2,7 @@ import gradio as gr
 import spacy  # noqa
 from transformers import pipeline
 
-# import os
-# os.environ['KMP_DUPLICATE_LIB_OK']='True'
-# import spacy
-
-# Change this according to what words should be corrected to
-SPELL_CORRECT_MIN_CHAR_DIFF = 2
-
-TOKENS2INT_ERROR_INT = 32202
-
-ONES = [
-    "zero", "one", "two", "three", "four", "five", "six", "seven", "eight",
-    "nine", "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen",
-    "sixteen", "seventeen", "eighteen", "nineteen",
-]
-
-CHAR_MAPPING = {
-    "-": " ",
-    "_": " ",
-    "and": " ",
-}
-# CHAR_MAPPING.update((str(i), word) for i, word in enumerate([" " + s + " " for s in ONES]))
-TOKEN_MAPPING = {
-    "and": " ",
-    "oh": "0",
-}
-
-
-def find_char_diff(a, b):
-    # Finds the character difference between two str objects by counting the occurences of every character. Not edit distance.
-    char_counts_a = {}
-    char_counts_b = {}
-    for char in a:
-        if char in char_counts_a.keys():
-            char_counts_a[char] += 1
-        else:
-            char_counts_a[char] = 1
-    for char in b:
-        if char in char_counts_b.keys():
-            char_counts_b[char] += 1
-        else:
-            char_counts_b[char] = 1
-    char_diff = 0
-    for i in char_counts_a:
-        if i in char_counts_b.keys():
-            char_diff += abs(char_counts_a[i] - char_counts_b[i])
-        else:
-            char_diff += char_counts_a[i]
-    return char_diff
-
-
-def tokenize(text):
-    text = text.lower()
-    # print(text)
-    text = replace_tokens(''.join(i for i in replace_chars(text)).split())
-    # print(text)
-    text = [i for i in text if i != ' ']
-    # print(text)
-    output = []
-    for word in text:
-        # print(word)
-        output.append(convert_word_to_int(word))
-    output = [i for i in output if i != ' ']
-    # print(output)
-    return output
-
-
-def detokenize(tokens):
-    return ' '.join(tokens)
-
-
-def replace_tokens(tokens, token_mapping=TOKEN_MAPPING):
-    return [token_mapping.get(tok, tok) for tok in tokens]
-
-
-def replace_chars(text, char_mapping=CHAR_MAPPING):
-    return [char_mapping.get(c, c) for c in text]
-
-
-def convert_word_to_int(in_word, numwords={}):
-    # Converts a single word/str into a single int
-    tens = ["", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety"]
-    scales = ["hundred", "thousand", "million", "billion", "trillion"]
-    if not numwords:
-        for idx, word in enumerate(ONES):
-            numwords[word] = idx
-        for idx, word in enumerate(tens):
-            numwords[word] = idx * 10
-        for idx, word in enumerate(scales):
-            numwords[word] = 10 ** (idx * 3 or 2)
-    if in_word in numwords:
-        # print(in_word)
-        # print(numwords[in_word])
-        return numwords[in_word]
-    try:
-        int(in_word)
-        return int(in_word)
-    except ValueError:
-        pass
-    # Spell correction using find_char_diff
-    char_diffs = [find_char_diff(in_word, i) for i in ONES + tens + scales]
-    min_char_diff = min(char_diffs)
-    if min_char_diff <= SPELL_CORRECT_MIN_CHAR_DIFF:
-        return char_diffs.index(min_char_diff)
-
-
-def tokens2int(tokens):
-    # Takes a list of tokens and returns a int representation of them
-    types = []
-    for i in tokens:
-        if i <= 9:
-            types.append(1)
-
-        elif i <= 90:
-            types.append(2)
-
-        else:
-            types.append(3)
-    # print(tokens)
-    if len(tokens) <= 3:
-        current = 0
-        for i, number in enumerate(tokens):
-            if i != 0 and types[i] < types[i - 1] and current != tokens[i - 1] and types[i - 1] != 3:
-                current += tokens[i] + tokens[i - 1]
-            elif current <= tokens[i] and current != 0:
-                current *= tokens[i]
-            elif 3 not in types and 1 not in types:
-                current = int(''.join(str(i) for i in tokens))
-                break
-            elif '111' in ''.join(str(i) for i in types) and 2 not in types and 3 not in types:
-                current = int(''.join(str(i) for i in tokens))
-                break
-            else:
-                current += number
-
-    elif 3 not in types and 2 not in types:
-        current = int(''.join(str(i) for i in tokens))
-
-    else:
-        """
-        double_list = []
-        current_double = []
-        double_type_list = []
-        for i in tokens:
-            if len(current_double) < 2:
-                current_double.append(i)
-            else:
-                double_list.append(current_double)
-                current_double = []
-        current_double = []
-        for i in types:
-            if len(current_double) < 2:
-                current_double.append(i)
-            else:
-                double_type_list.append(current_double)
-                current_double = []
-        print(double_type_list)
-        print(double_list)
-        current = 0
-        for i, type_double in enumerate(double_type_list):
-            if len(type_double) == 1:
-                current += double_list[i][0]
-            elif type_double[0] == type_double[1]:
-                current += int(str(double_list[i][0]) + str(double_list[i][1]))
-            elif type_double[0] > type_double[1]:
-                current += sum(double_list[i])
-            elif type_double[0] < type_double[1]:
-                current += double_list[i][0] * double_list[i][1]
-        #print(current)
-        """
-        count = 0
-        current = 0
-        for i, token in enumerate(tokens):
-            count += 1
-            if count == 2:
-                if types[i - 1] == types[i]:
-                    current += int(str(token) + str(tokens[i - 1]))
-                elif types[i - 1] > types[i]:
-                    current += tokens[i - 1] + token
-                else:
-                    current += tokens[i - 1] * token
-                count = 0
-            elif i == len(tokens) - 1:
-                current += token
-
-    return current
-
-
-def text2int(text):
-    # Wraps all of the functions up into one
-    return tokens2int(tokenize(text))
+from mathtext.nlutils import text2int
 
 
 sentiment = pipeline(task="sentiment-analysis", model="distilbert-base-uncased-finetuned-sst-2-english")
@@ -206,9 +17,9 @@ with gr.Blocks() as html_block:
 
     with gr.Tab("Text to integer"):
         inputs_text2int = [gr.Text(
-                    placeholder="Type a number as text or a sentence",
-                    label="Text to process",
-                    value="forty two")]
+            placeholder="Type a number as text or a sentence",
+            label="Text to process",
+            value="forty two")]
 
         outputs_text2int = gr.Textbox(label="Output integer")
 

nlutils.py

@@ -0,0 +1,327 @@
+import spacy  # noqa
+import time
+from editdistance import eval as edit_dist
+
+# import os
+# os.environ['KMP_DUPLICATE_LIB_OK']='True'
+# import spacy
+
+# Change this according to what words should be corrected to
+SPELL_CORRECT_MIN_CHAR_DIFF = 2
+
+TOKENS2INT_ERROR_INT = 32202
+
+ONES = [
+    "zero", "one", "two", "three", "four", "five", "six", "seven", "eight",
+    "nine", "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen",
+    "sixteen", "seventeen", "eighteen", "nineteen",
+]
+
+CHAR_MAPPING = {
+    "-": " ",
+    "_": " ",
+    "and": " ",
+}
+# CHAR_MAPPING.update((str(i), word) for i, word in enumerate([" " + s + " " for s in ONES]))
+TOKEN_MAPPING = {
+    "and": " ",
+    "oh": "0",
+}
+
+
+def find_char_diff(a, b):
+    # Finds the character difference between two str objects by counting the occurences of every character. Not edit distance.
+    char_counts_a = {}
+    char_counts_b = {}
+    for char in a:
+        if char in char_counts_a.keys():
+            char_counts_a[char] += 1
+        else:
+            char_counts_a[char] = 1
+    for char in b:
+        if char in char_counts_b.keys():
+            char_counts_b[char] += 1
+        else:
+            char_counts_b[char] = 1
+    char_diff = 0
+    for i in char_counts_a:
+        if i in char_counts_b.keys():
+            char_diff += abs(char_counts_a[i] - char_counts_b[i])
+        else:
+            char_diff += char_counts_a[i]
+    return char_diff
+
+
+def tokenize(text):
+    text = text.lower()
+    # print(text)
+    text = replace_tokens(''.join(i for i in replace_chars(text)).split())
+    # print(text)
+    text = [i for i in text if i != ' ']
+    # print(text)
+    output = []
+    for word in text:
+        # print(word)
+        output.append(convert_word_to_int(word))
+    output = [i for i in output if i != ' ']
+    # print(output)
+    return output
+
+
+def detokenize(tokens):
+    return ' '.join(tokens)
+
+
+def replace_tokens(tokens, token_mapping=TOKEN_MAPPING):
+    return [token_mapping.get(tok, tok) for tok in tokens]
+
+
+def replace_chars(text, char_mapping=CHAR_MAPPING):
+    return [char_mapping.get(c, c) for c in text]
+
+
+def convert_word_to_int(in_word, numwords={}):
+    # Converts a single word/str into a single int
+    tens = ["", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety"]
+    scales = ["hundred", "thousand", "million", "billion", "trillion"]
+    if not numwords:
+        for idx, word in enumerate(ONES):
+            numwords[word] = idx
+        for idx, word in enumerate(tens):
+            numwords[word] = idx * 10
+        for idx, word in enumerate(scales):
+            numwords[word] = 10 ** (idx * 3 or 2)
+    if in_word in numwords:
+        # print(in_word)
+        # print(numwords[in_word])
+        return numwords[in_word]
+    try:
+        int(in_word)
+        return int(in_word)
+    except ValueError:
+        pass
+    # Spell correction using find_char_diff
+    char_diffs = [find_char_diff(in_word, i) for i in ONES + tens + scales]
+    min_char_diff = min(char_diffs)
+    if min_char_diff <= SPELL_CORRECT_MIN_CHAR_DIFF:
+        return char_diffs.index(min_char_diff)
+
+
+def tokens2int(tokens):
+    # Takes a list of tokens and returns a int representation of them
+    types = []
+    for i in tokens:
+        if i <= 9:
+            types.append(1)
+
+        elif i <= 90:
+            types.append(2)
+
+        else:
+            types.append(3)
+    # print(tokens)
+    if len(tokens) <= 3:
+        current = 0
+        for i, number in enumerate(tokens):
+            if i != 0 and types[i] < types[i - 1] and current != tokens[i - 1] and types[i - 1] != 3:
+                current += tokens[i] + tokens[i - 1]
+            elif current <= tokens[i] and current != 0:
+                current *= tokens[i]
+            elif 3 not in types and 1 not in types:
+                current = int(''.join(str(i) for i in tokens))
+                break
+            elif '111' in ''.join(str(i) for i in types) and 2 not in types and 3 not in types:
+                current = int(''.join(str(i) for i in tokens))
+                break
+            else:
+                current += number
+
+    elif 3 not in types and 2 not in types:
+        current = int(''.join(str(i) for i in tokens))
+
+    else:
+        """
+        double_list = []
+        current_double = []
+        double_type_list = []
+        for i in tokens:
+            if len(current_double) < 2:
+                current_double.append(i)
+            else:
+                double_list.append(current_double)
+                current_double = []
+        current_double = []
+        for i in types:
+            if len(current_double) < 2:
+                current_double.append(i)
+            else:
+                double_type_list.append(current_double)
+                current_double = []
+        print(double_type_list)
+        print(double_list)
+        current = 0
+        for i, type_double in enumerate(double_type_list):
+            if len(type_double) == 1:
+                current += double_list[i][0]
+            elif type_double[0] == type_double[1]:
+                current += int(str(double_list[i][0]) + str(double_list[i][1]))
+            elif type_double[0] > type_double[1]:
+                current += sum(double_list[i])
+            elif type_double[0] < type_double[1]:
+                current += double_list[i][0] * double_list[i][1]
+        # print(current)
+        """
+        count = 0
+        current = 0
+        for i, token in enumerate(tokens):
+            count += 1
+            if count == 2:
+                if types[i - 1] == types[i]:
+                    current += int(str(token) + str(tokens[i - 1]))
+                elif types[i - 1] > types[i]:
+                    current += tokens[i - 1] + token
+                else:
+                    current += tokens[i - 1] * token
+                count = 0
+            elif i == len(tokens) - 1:
+                current += token
+
+    return current
+
+
+def text2int(text):
+    # Wraps all of the functions up into one
+    return tokens2int(tokenize(text))
+
+###############################################
+# Vish editdistance approach doesn't halt
+
+
+def lev_dist(a, b):
+    '''
+    This function will calculate the levenshtein distance between two input
+    strings a and b
+
+    params:
+        a (String) : The first string you want to compare
+        b (String) : The second string you want to compare
+
+    returns:
+        This function will return the distance between string a and b.
+
+    example:
+        a = 'stamp'
+        b = 'stomp'
+        lev_dist(a,b)
+        >> 1.0
+    '''
+    if not isinstance(a, str) and isinstance(b, str):
+        raise ValueError(f"lev_dist() requires 2 strings not lev_dist({repr(a)}, {repr(b)}")
+    if a == b:
+        return 0
+
+    def min_dist(s1, s2):
+
+        print(f"{a[s1]}s1{b[s2]}s2 ", end='')
+        if s1 >= len(a) or s2 >= len(b):
+            return len(a) - s1 + len(b) - s2
+
+        # no change required
+        if a[s1] == b[s2]:
+            return min_dist(s1 + 1, s2 + 1)
+
+        return 1 + min(
+            min_dist(s1, s2 + 1),      # insert character
+            min_dist(s1 + 1, s2),      # delete character
+            min_dist(s1 + 1, s2 + 1),  # replace character
+        )
+
+    dist = min_dist(0, 0)
+    print(f"\n  lev_dist({a}, {b}) => {dist}")
+    return dist
+
+
+def correct_number_text(text):
+    """ Convert an English str containing number words with possibly incorrect spellings into an int
+
+    >>> robust_text2int("too")
+    2
+    >>> robust_text2int("fore")
+    4
+    >>> robust_text2int("1 2 tree")
+    123
+    """
+    words = {
+        "zero": 0,
+        "one": 1,
+        "two": 2,
+        "three": 3,
+        "four": 4,
+        "five": 5,
+        "six": 6,
+        "seven": 7,
+        "eight": 8,
+        "nine": 9,
+        "ten": 10,
+        "eleven": 11,
+        "twelve": 12,
+        "thirteen": 13,
+        "fourteen": 14,
+        "fifteen": 15,
+        "sixteen": 16,
+        "seventeen": 17,
+        "eighteen": 18,
+        "nineteen": 19,
+        "score": 20,
+        "twenty": 20,
+        "thirty": 30,
+        "forty": 40,
+        "fifty": 50,
+        "sixty": 60,
+        "seventy": 70,
+        "eighty": 80,
+        "ninety": 90,
+        "hundred": 100,
+        "thousand": 1000,
+        "million": 1000000,
+        "billion": 1000000000,
+    }
+
+    text = text.lower()
+    text_words = text.replace("-", " ").split()
+    corrected_words = []
+    for text_word in text_words:
+        if text_word not in words:
+            print(f"{text_word} not in words")
+            if not isinstance(text_word, str):
+                return TOKENS2INT_ERROR_INT
+            t0 = time.time()
+            min_dist = len(text_word)
+            correct_spelling = None
+            for word in words:
+                dist = edit_dist(word, text_word)
+                if dist < min_dist:
+                    correct_spelling = word
+                    min_dist = dist
+            corrected_words.append(correct_spelling)
+            t1 = time.time()
+            print(f"{text_word} dt:{t1-t0}")
+        else:
+            corrected_words.append(text_word)
+
+    corrected_text = " ".join(corrected_words)
+
+    print(corrected_text)
+    return corrected_text
+
+    # From hereon, we can use text2int
+    # TODO
+
+
+def robust_text2int(text):
+    """ Correct spelling of number words in text before using text2int """
+    try:
+        return tokens2int(tokenize(correct_number_text(text)))
+    except Exception as e:
+        print(e)
+    return TOKENS2INT_ERROR_INT