First commit

app.py

@@ -0,0 +1,31 @@
+import torch
+import gradio as gr
+from language_bpe import BPETokenizer
+
+tokenizer = BPETokenizer()
+tokenizer.load('models/english_5000.model')
+
+def inference(input_text):
+    tokens = tokenizer.encode_ordinary(input_text)
+    
+    return tokens
+
+title = "A bilingual tokenizer build using opus and wikipedia data"
+description = "A simple Gradio interface to see tokenization of Hindi and English(Hinglish) text"
+examples = [["He walked into the basement with the horror movie from the night before playing in his head."], 
+           ["Henry couldn't decide if he was an auto mechanic or a priest."], 
+           ["Poison ivy grew through the fence they said was impenetrable."], 
+           ]
+demo = gr.Interface(
+    inference, 
+    inputs = [
+        gr.Textbox(label="Enter any sentence in Hindi, English or both language", type="text"),
+        ], 
+    outputs = [
+        gr.Textbox(label="Output", type="text")
+        ],
+    title = title,
+    description = description,
+    examples = examples,
+)
+demo.launch()

language_bpe/__init__.py

@@ -0,0 +1,2 @@
+from .base import Tokenizer
+from .bpe_tokenizer import BPETokenizer

language_bpe/base.py

@@ -0,0 +1,141 @@
+import unicodedata
+
+def get_stats(ids, counts=None):
+    counts = {} if counts is None else counts
+    for pair in zip(ids, ids[1:]):
+        counts[pair] = counts.get(pair, 0) + 1
+    return counts
+
+def merge(ids, pair, idx):
+    newids = []
+    i = 0
+    while i < len(ids):
+        if ids[i] == pair[0] and i < len(ids) - 1 and ids[i+1] == pair[1]:
+            newids.append(idx)
+            i += 2
+        else:
+            newids.append(ids[i])
+            i += 1
+    return newids
+
+def merge_hindi(ids, pair, idx):
+    newids = []
+    i = 0
+    while i < len(ids):
+        # if not at the very last position AND the pair matches, replace it
+        if ids[i] == pair[0] and i < len(ids) - 2 and ids[i+1] == pair[1] and ids[i+2] == pair[2]:
+            newids.append(idx)
+            i += 3
+        else:
+            newids.append(ids[i])
+            i += 1
+    return newids
+
+def replace_control_characters(s):
+    chars = []
+    for ch in s:
+        if unicodedata.category(ch)[0] != "C":
+            chars.append(ch)
+        else:
+            chars.append(f"\\u{ord(ch):04x}")
+    return "".join(chars)
+
+def render_token(t):
+    s = t.decode('utf-8', errors='replace')
+    s = replace_control_characters(s)
+    return s
+
+class Tokenizer:
+    def __init__(self):
+        self.merges = {}
+        self.pattern = ""
+        self.special_tokens = {}
+        self.vocab = self._build_vocab()
+
+    def build(self, text, vocab_size, verbose=False):
+        raise NotImplementedError
+
+    def encode(self, text):
+        raise NotImplementedError
+
+    def decode(self, ids):
+        raise NotImplementedError
+
+    def _build_vocab(self):
+        # vocab is simply and deterministically derived from merges
+        vocab = {idx: bytes([idx]) for idx in range(256)}
+        for (p0, p1), idx in self.merges.items():
+            vocab[idx] = vocab[p0] + vocab[p1]
+        for special, idx in self.special_tokens.items():
+            vocab[idx] = special.encode("utf-8")
+        return vocab
+
+    def save(self, file_prefix):
+        """
+        Saves two files: file_prefix.vocab and file_prefix.model
+        This is inspired (but not equivalent to!) sentencepiece's model saving:
+        - model file is the critical one, intended for load()
+        - vocab file is just a pretty printed version for human inspection only
+        """
+        # write the model: to be used in load() later
+        model_file = file_prefix + ".model"
+        with open(model_file, 'w') as f:
+            # write the version, pattern and merges, that's all that's needed
+            f.write("minbpe v1\n")
+            f.write(f"{self.pattern}\n")
+            # write the special tokens, first the number of them, then each one
+            f.write(f"{len(self.special_tokens)}\n")
+            for special, idx in self.special_tokens.items():
+                f.write(f"{special} {idx}\n")
+            # the merges dict
+            for idx1, idx2 in self.merges:
+                f.write(f"{idx1} {idx2}\n")
+        # write the vocab: for the human to look at
+        vocab_file = file_prefix + ".vocab"
+        inverted_merges = {idx: pair for pair, idx in self.merges.items()}
+        with open(vocab_file, "w", encoding="utf-8") as f:
+            for idx, token in self.vocab.items():
+                # note: many tokens may be partial utf-8 sequences
+                # and cannot be decoded into valid strings. Here we're using
+                # errors='replace' to replace them with the replacement char �.
+                # this also means that we couldn't possibly use .vocab in load()
+                # because decoding in this way is a lossy operation!
+                s = render_token(token)
+                # find the children of this token, if any
+                if idx in inverted_merges:
+                    # if this token has children, render it nicely as a merge
+                    idx0, idx1 = inverted_merges[idx]
+                    s0 = render_token(self.vocab[idx0])
+                    s1 = render_token(self.vocab[idx1])
+                    f.write(f"[{s0}][{s1}] -> [{s}] {idx}\n")
+                else:
+                    # otherwise this is leaf token, just print it
+                    # (this should just be the first 256 tokens, the bytes)
+                    f.write(f"[{s}] {idx}\n")
+
+    def load(self, model_file):
+        """Inverse of save() but only for the model file"""
+        assert model_file.endswith(".model")
+        # read the model file
+        merges = {}
+        special_tokens = {}
+        idx = 256
+        with open(model_file, 'r', encoding="utf-8") as f:
+            # read the version
+            version = f.readline().strip()
+            assert version == "minbpe v1"
+            # read the pattern
+            self.pattern = f.readline().strip()
+            # read the special tokens
+            num_special = int(f.readline().strip())
+            for _ in range(num_special):
+                special, special_idx = f.readline().strip().split()
+                special_tokens[special] = int(special_idx)
+            # read the merges
+            for line in f:
+                idx1, idx2 = map(int, line.split())
+                merges[(idx1, idx2)] = idx
+                idx += 1
+        self.merges = merges
+        self.special_tokens = special_tokens
+        self.vocab = self._build_vocab()

language_bpe/bpe_tokenizer.py

@@ -0,0 +1,191 @@
+import sys
+import regex as re
+from tqdm import tqdm
+from .base import Tokenizer, get_stats, merge, merge_hindi
+
+GPT4_SPLIT_PATTERN = r"""'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]++[\r\n]*|\s*[\r\n]|\s+(?!\S)|\s+"""
+
+class BPETokenizer(Tokenizer):
+
+    def __init__(self, pattern=None, word_pattern = None):
+        super().__init__()
+        self.pattern = GPT4_SPLIT_PATTERN if pattern is None else pattern
+        self.compiled_pattern = re.compile(self.pattern)
+        self.word_pattern = None
+        self.compiled_pattern_word = None
+        if word_pattern:
+            self.word_pattern = word_pattern
+            self.compiled_pattern_word = re.compile(self.word_pattern)
+        self.special_tokens = {}
+        self.inverse_special_tokens = {}
+
+    def build(self, text, vocab_size, verbose=False):
+
+        text_chunks = re.findall(self.compiled_pattern, text)
+
+        if self.compiled_pattern_word:
+            print("Spliting hindi words")
+            text_chunks_words = []
+            for chunk in tqdm(text_chunks):
+                element_chunks = re.findall(self.compiled_pattern_word, chunk)
+                if element_chunks == []:
+                    text_chunks_words.append(chunk) 
+                else:
+                    text_chunks_words.extend(element_chunks[0]) 
+            text_chunks = text_chunks_words
+        
+        # input text preprocessing
+        ids = [list(ch.encode("utf-8")) for ch in text_chunks]
+
+        merges = {}
+        vocab = {idx: bytes([idx]) for idx in range(256)}
+        vocab.update({idx: bytes(list(chr(value).encode('utf-8'))) for idx,value in zip(range(256, 384), range(2304, 2432))})
+
+        print("Merging hindi characters in single token")
+        for index in tqdm(range(256, 384)):
+            pair = list(vocab[index])
+            ids = [merge_hindi(chunk_ids, pair, index) for chunk_ids in ids]
+        
+        num_merges = vocab_size - 384
+
+        original_length = len([x for xs in ids for x in xs])
+
+        print("Building BPE")
+        for i in tqdm(range(num_merges), file=sys.stdout):
+            # count the number of times every consecutive pair appears
+            stats = {}
+            for chunk_ids in ids:
+                # passing in stats will update it in place, adding up counts
+                get_stats(chunk_ids, stats)
+            # find the pair with the highest count
+            pair = max(stats, key=stats.get)
+            # mint a new token: assign it the next available id
+            idx = 384 + i
+            # replace all occurrences of pair in ids with idx
+            ids = [merge(chunk_ids, pair, idx) for chunk_ids in ids]
+            # save the merge
+            merges[pair] = idx
+            vocab[idx] = vocab[pair[0]] + vocab[pair[1]]
+            # prints
+            if verbose:
+                try:
+                    tqdm.write(f"merge {i+1}/{num_merges}: {pair} -> {idx} ({vocab[idx].decode('utf-8')}) had {stats[pair]} occurrences")
+                except Exception as e:
+                    tqdm.write(f"merge {i+1}/{num_merges}: {pair} -> {idx} ({vocab[idx]}) had {stats[pair]} occurrences")
+        
+        lenght_after_merging = len([x for xs in ids for x in xs])
+
+        print(f'Compression ratio: {original_length/lenght_after_merging}')
+
+        # save class variables
+        self.merges = merges # used in encode()
+        self.vocab = vocab   # used in decode()
+
+    def register_special_tokens(self, special_tokens):
+        # special_tokens is a dictionary of str -> int
+        # example: {"<|endoftext|>": 100257}
+        self.special_tokens = special_tokens
+        self.inverse_special_tokens = {v: k for k, v in special_tokens.items()}
+
+    def decode(self, ids):
+        # given ids (list of integers), return Python string
+        part_bytes = []
+        for idx in ids:
+            if idx in self.vocab:
+                part_bytes.append(self.vocab[idx])
+            elif idx in self.inverse_special_tokens:
+                part_bytes.append(self.inverse_special_tokens[idx].encode("utf-8"))
+            else:
+                raise ValueError(f"invalid token id: {idx}")
+        text_bytes = b"".join(part_bytes)
+        text = text_bytes.decode("utf-8", errors="replace")
+        return text
+
+    def _encode_chunk(self, ids):
+        # return the token ids
+        # let's begin. first, convert all bytes to integers in range 0..255
+        while len(ids) >= 2:
+            # find the pair with the lowest merge index
+            stats = get_stats(ids)
+            pair = min(stats, key=lambda p: self.merges.get(p, float("inf")))
+            # subtle: if there are no more merges available, the key will
+            # result in an inf for every single pair, and the min will be
+            # just the first pair in the list, arbitrarily
+            # we can detect this terminating case by a membership check
+            if pair not in self.merges:
+                break # nothing else can be merged anymore
+            # otherwise let's merge the best pair (lowest merge index)
+            idx = self.merges[pair]
+            ids = merge(ids, pair, idx)
+        return ids
+
+    def encode_ordinary(self, text):
+        """Encoding that ignores any special tokens."""
+        # split text into chunks of text by categories defined in regex pattern
+        text_chunks = re.findall(self.compiled_pattern, text)
+        if self.compiled_pattern_word:
+            print("Spliting hindi words")
+            text_chunks_words = []
+            for chunk in tqdm(text_chunks):
+                element_chunks = re.findall(self.compiled_pattern_word, chunk)
+                if element_chunks == []:
+                    text_chunks_words.append(chunk) 
+                else:
+                    text_chunks_words.extend(element_chunks[0]) 
+            text_chunks = text_chunks_words
+        # all chunks of text are encoded separately, then results are joined
+        ids_list = []
+        for chunk in text_chunks:
+            chunk_bytes = chunk.encode("utf-8") # raw bytes
+            ids = list(chunk_bytes)
+            vocab = {idx: bytes([idx]) for idx in range(256)}
+            vocab.update({idx: bytes(list(chr(value).encode('utf-8'))) for idx,value in zip(range(256, 384), range(2304, 2432))})
+            for index in tqdm(range(256, 384)):
+                pair = list(vocab[index])
+                ids = [merge_hindi(chunk_ids, pair, index) for chunk_ids in ids]
+            chunk_ids = self._encode_chunk(ids)
+            ids_list.extend(chunk_ids)
+        return ids_list
+
+    def encode(self, text, allowed_special="none_raise"):
+        """
+        Unlike encode_ordinary, this function handles special tokens.
+        allowed_special: can be "all"|"none"|"none_raise" or a custom set of special tokens
+        if none_raise, then an error is raised if any special token is encountered in text
+        this is the default tiktoken behavior right now as well
+        any other behavior is either annoying, or a major footgun
+        """
+        # decode the user desire w.r.t. handling of special tokens
+        special = None
+        if allowed_special == "all":
+            special = self.special_tokens
+        elif allowed_special == "none":
+            special = {}
+        elif allowed_special == "none_raise":
+            special = {}
+            assert all(token not in text for token in self.special_tokens)
+        elif isinstance(allowed_special, set):
+            special = {k: v for k, v in self.special_tokens.items() if k in allowed_special}
+        else:
+            raise ValueError(f"allowed_special={allowed_special} not understood")
+        if not special:
+            # shortcut: if no special tokens, just use the ordinary encoding
+            return self.encode_ordinary(text)
+        # otherwise, we have to be careful with potential special tokens in text
+        # we handle special tokens by splitting the text
+        # based on the occurrence of any exact match with any of the special tokens
+        # we can use re.split for this. note that surrounding the pattern with ()
+        # makes it into a capturing group, so the special tokens will be included
+        special_pattern = "(" + "|".join(re.escape(k) for k in special) + ")"
+        special_chunks = re.split(special_pattern, text)
+        # now all the special characters are separated from the rest of the text
+        # all chunks of text are encoded separately, then results are joined
+        ids = []
+        for part in special_chunks:
+            if part in special:
+                # this is a special token, encode it separately as a special case
+                ids.append(special[part])
+            else:
+                # this is an ordinary sequence, encode it normally
+                ids.extend(self.encode_ordinary(part))
+        return ids

models/english_5000.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2b8dbed9bf76124c7660f13499936c2015504e9e29c1a4e4be3af3bbfbb51f0
+size 38534

models/hindi_5000.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b844ea0a4156cc551b5bfdeeadb65c6ff77deb7cbf609518195f521cc6cac66
+size 38844

models/hinglish_5000.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f53512686a6e268b23806738311e5589e604c923ed412ccf9cda7c8a041acb0c
+size 38540