nlutils.py

import spacy  # noqa
import time
from editdistance import eval as edit_dist

# import os
# os.environ['KMP_DUPLICATE_LIB_OK']='True'
# import spacy

# Change this according to what words should be corrected to
SPELL_CORRECT_MIN_CHAR_DIFF = 2

TOKENS2INT_ERROR_INT = 32202

ONES = [
    "zero", "one", "two", "three", "four", "five", "six", "seven", "eight",
    "nine", "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen",
    "sixteen", "seventeen", "eighteen", "nineteen",
]

CHAR_MAPPING = {
    "-": " ",
    "_": " ",
    "and": " ",
}
# CHAR_MAPPING.update((str(i), word) for i, word in enumerate([" " + s + " " for s in ONES]))
TOKEN_MAPPING = {
    "and": " ",
    "oh": "0",
}


def find_char_diff(a, b):
    # Finds the character difference between two str objects by counting the occurences of every character. Not edit distance.
    char_counts_a = {}
    char_counts_b = {}
    for char in a:
        if char in char_counts_a.keys():
            char_counts_a[char] += 1
        else:
            char_counts_a[char] = 1
    for char in b:
        if char in char_counts_b.keys():
            char_counts_b[char] += 1
        else:
            char_counts_b[char] = 1
    char_diff = 0
    for i in char_counts_a:
        if i in char_counts_b.keys():
            char_diff += abs(char_counts_a[i] - char_counts_b[i])
        else:
            char_diff += char_counts_a[i]
    return char_diff


def tokenize(text):
    text = text.lower()
    # print(text)
    text = replace_tokens(''.join(i for i in replace_chars(text)).split())
    # print(text)
    text = [i for i in text if i != ' ']
    # print(text)
    output = []
    for word in text:
        # print(word)
        output.append(convert_word_to_int(word))
    output = [i for i in output if i != ' ']
    # print(output)
    return output


def detokenize(tokens):
    return ' '.join(tokens)


def replace_tokens(tokens, token_mapping=TOKEN_MAPPING):
    return [token_mapping.get(tok, tok) for tok in tokens]


def replace_chars(text, char_mapping=CHAR_MAPPING):
    return [char_mapping.get(c, c) for c in text]


def convert_word_to_int(in_word, numwords={}):
    # Converts a single word/str into a single int
    tens = ["", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety"]
    scales = ["hundred", "thousand", "million", "billion", "trillion"]
    if not numwords:
        for idx, word in enumerate(ONES):
            numwords[word] = idx
        for idx, word in enumerate(tens):
            numwords[word] = idx * 10
        for idx, word in enumerate(scales):
            numwords[word] = 10 ** (idx * 3 or 2)
    if in_word in numwords:
        # print(in_word)
        # print(numwords[in_word])
        return numwords[in_word]
    try:
        int(in_word)
        return int(in_word)
    except ValueError:
        pass
    # Spell correction using find_char_diff
    char_diffs = [find_char_diff(in_word, i) for i in ONES + tens + scales]
    min_char_diff = min(char_diffs)
    if min_char_diff <= SPELL_CORRECT_MIN_CHAR_DIFF:
        return char_diffs.index(min_char_diff)


def tokens2int(tokens):
    # Takes a list of tokens and returns a int representation of them
    types = []
    for i in tokens:
        if i <= 9:
            types.append(1)

        elif i <= 90:
            types.append(2)

        else:
            types.append(3)
    # print(tokens)
    if len(tokens) <= 3:
        current = 0
        for i, number in enumerate(tokens):
            if i != 0 and types[i] < types[i - 1] and current != tokens[i - 1] and types[i - 1] != 3:
                current += tokens[i] + tokens[i - 1]
            elif current <= tokens[i] and current != 0:
                current *= tokens[i]
            elif 3 not in types and 1 not in types:
                current = int(''.join(str(i) for i in tokens))
                break
            elif '111' in ''.join(str(i) for i in types) and 2 not in types and 3 not in types:
                current = int(''.join(str(i) for i in tokens))
                break
            else:
                current += number

    elif 3 not in types and 2 not in types:
        current = int(''.join(str(i) for i in tokens))

    else:
        """
        double_list = []
        current_double = []
        double_type_list = []
        for i in tokens:
            if len(current_double) < 2:
                current_double.append(i)
            else:
                double_list.append(current_double)
                current_double = []
        current_double = []
        for i in types:
            if len(current_double) < 2:
                current_double.append(i)
            else:
                double_type_list.append(current_double)
                current_double = []
        print(double_type_list)
        print(double_list)
        current = 0
        for i, type_double in enumerate(double_type_list):
            if len(type_double) == 1:
                current += double_list[i][0]
            elif type_double[0] == type_double[1]:
                current += int(str(double_list[i][0]) + str(double_list[i][1]))
            elif type_double[0] > type_double[1]:
                current += sum(double_list[i])
            elif type_double[0] < type_double[1]:
                current += double_list[i][0] * double_list[i][1]
        # print(current)
        """
        count = 0
        current = 0
        for i, token in enumerate(tokens):
            count += 1
            if count == 2:
                if types[i - 1] == types[i]:
                    current += int(str(token) + str(tokens[i - 1]))
                elif types[i - 1] > types[i]:
                    current += tokens[i - 1] + token
                else:
                    current += tokens[i - 1] * token
                count = 0
            elif i == len(tokens) - 1:
                current += token

    return current


def text2int(text):
    # Wraps all of the functions up into one
    return tokens2int(tokenize(text))

###############################################
# Vish editdistance approach doesn't halt


def lev_dist(a, b):
    '''
    This function will calculate the levenshtein distance between two input
    strings a and b

    params:
        a (String) : The first string you want to compare
        b (String) : The second string you want to compare

    returns:
        This function will return the distance between string a and b.

    example:
        a = 'stamp'
        b = 'stomp'
        lev_dist(a,b)
        >> 1.0
    '''
    if not isinstance(a, str) and isinstance(b, str):
        raise ValueError(f"lev_dist() requires 2 strings not lev_dist({repr(a)}, {repr(b)}")
    if a == b:
        return 0

    def min_dist(s1, s2):

        print(f"{a[s1]}s1{b[s2]}s2 ", end='')
        if s1 >= len(a) or s2 >= len(b):
            return len(a) - s1 + len(b) - s2

        # no change required
        if a[s1] == b[s2]:
            return min_dist(s1 + 1, s2 + 1)

        return 1 + min(
            min_dist(s1, s2 + 1),      # insert character
            min_dist(s1 + 1, s2),      # delete character
            min_dist(s1 + 1, s2 + 1),  # replace character
        )

    dist = min_dist(0, 0)
    print(f"\n  lev_dist({a}, {b}) => {dist}")
    return dist


def correct_number_text(text):
    """ Convert an English str containing number words with possibly incorrect spellings into an int

    >>> robust_text2int("too")
    2
    >>> robust_text2int("fore")
    4
    >>> robust_text2int("1 2 tree")
    123
    """
    words = {
        "zero": 0,
        "one": 1,
        "two": 2,
        "three": 3,
        "four": 4,
        "five": 5,
        "six": 6,
        "seven": 7,
        "eight": 8,
        "nine": 9,
        "ten": 10,
        "eleven": 11,
        "twelve": 12,
        "thirteen": 13,
        "fourteen": 14,
        "fifteen": 15,
        "sixteen": 16,
        "seventeen": 17,
        "eighteen": 18,
        "nineteen": 19,
        "score": 20,
        "twenty": 20,
        "thirty": 30,
        "forty": 40,
        "fifty": 50,
        "sixty": 60,
        "seventy": 70,
        "eighty": 80,
        "ninety": 90,
        "hundred": 100,
        "thousand": 1000,
        "million": 1000000,
        "billion": 1000000000,
    }

    text = text.lower()
    text_words = text.replace("-", " ").split()
    corrected_words = []
    for text_word in text_words:
        if text_word not in words:
            print(f"{text_word} not in words")
            if not isinstance(text_word, str):
                return TOKENS2INT_ERROR_INT
            t0 = time.time()
            min_dist = len(text_word)
            correct_spelling = None
            for word in words:
                dist = edit_dist(word, text_word)
                if dist < min_dist:
                    correct_spelling = word
                    min_dist = dist
            corrected_words.append(correct_spelling)
            t1 = time.time()
            print(f"{text_word} dt:{t1-t0}")
        else:
            corrected_words.append(text_word)

    corrected_text = " ".join(corrected_words)

    print(corrected_text)
    return corrected_text

    # From hereon, we can use text2int
    # TODO


def robust_text2int(text):
    """ Correct spelling of number words in text before using text2int """
    try:
        return tokens2int(tokenize(correct_number_text(text)))
    except Exception as e:
        print(e)
    return TOKENS2INT_ERROR_INT