Upload tokenizer

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+subtoken_reference.json filter=lfs diff=lfs merge=lfs -text
+merges.json filter=lfs diff=lfs merge=lfs -text

chars.txt

@@ -0,0 +1 @@
+באדגוהחזיטכךםלןמסנףעץפקצשרתꙀꙁꙂꙃꙄꙅꙆꙇꙈꙉꙊꙋꙌꙍꙎꙏꙐꙑꙒꙓꙔꙕꙖꙗꙘꙙꙚꙛꙜꙝꙞꙟꙠꙡꙢꙣꙤꙥꙦꙧꙨꙩꙪꙫꙬꙭꙮ꙯꙰꙱꙲꙳ꙴꙵꙶꙷꙸꙹꙺꙻ꙼꙽꙾ꙿꚀꚁꚂꚃꚄꚅꚆꚇꚈꚉꚊꚋꚌꚍꚎꚏꚐꚑꚒꚓꚔꚕꚖꚗꚟЀЁЂЃЄЅІЇЈЉЊЋЌЍЎЏАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюяѐёђѓєѕіїјљњћќѝўџѠѡѢѣѤѥѦѧѨѩѪѫѬѭѮѯѰѱѲѳѴѵѶѷѸѹѺѻѼѽѾѿҀҁ҂҈҉ҊҋҌҍҎҏҐґҒғҔҕҖҗҘҙҚқҜҝҞҟҠҡҢңҤҥҦҧҨҩҪҫҬҭҮүҰұҲҳҴҵҶҷҸҹҺһҼҽҾҿӀӁӂӃӄӅӆӇӈӉӊӋӌӍӎӏӐӑӒӓӔӕӖӗӘәӚӛӜӝӞӟӠӡӢӣӤӥӦӧӨөӪӫӬӭӮӯӰӱӲӳӴӵӶӷӸӹӺӻӼӽӾӿԀԁԂԃԄԅԆԇԈԉԊԋԌԍԎԏԐԑԒԓԔԕԖԗԘԙԚԛԜԝԞԟԠԡԢԣԤԥԦԧἀἁἂἃἄἅἆἇἈἉἊἋἌἍἎἏἐἑἒἓἔἕἘἙἚἛἜἝἠἡἢἣἤἥἦἧἨἩἪἫἬἭἮἯἰἱἲἳἴἵἶἷἸἹἺἻἼἽἾἿὀὁὂὃὄὅὈὉὊὋὌὍὐὑὒὓὔὕὖὗὙὛὝὟὠὡὢὣὤὥὦὧὨὩὪὫὬὭὮὯὰάὲέὴήὶίὸόὺύὼώᾀᾁᾂᾃᾄᾅᾆᾇᾈᾉᾊᾋᾌᾍᾎᾏᾐᾑᾒᾓᾔᾕᾖᾗᾘᾙᾚᾛᾜᾝᾞᾟᾠᾡᾢᾣᾤᾥᾦᾧᾨᾩᾪᾫᾬᾭᾮᾯᾰᾱᾲᾳᾴᾶᾷᾸᾹᾺΆᾼ᾽ι᾿῀῁ῂῃῄῆῇῈΈῊΉῌ῍῎῏ῐῑῒΐῖῗῘῙῚΊ῝῞῟ῠῡῢΰῤῥῦῧῨῩῪΎῬ῭΅`ῲῳῴῶῷῸΌῺΏῼ´῾ϼϺϻϷϸ϶ϵϳϲϹϱϰϮϯϬϭϪϫϨϩϦϧϤϥϢϣϠϡϞϟϜϝϚϛϘϙϗϖϕϔϓϒϑϐΏώΎύΌόΫϋΪϊΩωΨψΧχΦφΥυΤτσΣςΡρΠπΟοΞξΝνΜμΛλΚκΙιΘθϴΗηΖζΕεΔδΓγΒβΑαΰΊίΉήΈέΆάΐͽϿͼϾͻϽȀȁȂȃȄȅȆȇȈȉȊȋȌȍȎȏȐȑȒȓȔȕȖȗȴȵȶȘșȚțǍǎǏǐǑǒǓǔǕǖǗǘǙǚǛǜǶǷǝǞǟǠǡǢǣǤǥǦǧǨǩǪǫǬǭǮǯǰDZDzdzǴǵǸǹǺǻǼǽǾǿƀƂƃƄƅƇƈƋƌƍƑƒƕƘƙƚƛƞƠơƢƣƤƥƧƨƪƫƬƭƯưƳƴƵƶƸƹƺƻƼƽƾƿƎƁƆƉƊƏƐƓƔƗƖƜƝƟƦƩƮƱƲƷɃȽȠȜȝȞȟȡȢȣȤȥȦȧȨȩȷȸȹȻȼȿɀɁɂɆɇɈɉɊɋɌɍɎɏɄɅȺȾȪȫȬȭȮȯȰȱȲȳDŽDždžLJLjljNJNjnjḂḃḊḋḞḟṀṁṖṗṠṡṪṫẀẁẂẃẄẅẛỲỳabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿچ‌گژءأآؤائکبتحجدخرذسزصشطضعظغفقملهنوًيیَُِّٔثپőű征安伎†ả箭ľ@ł介Œ四ởủ藩ɨžāąčḏđęĩĭ°ĽŁك則ō£őřɟ͡ũ火楽ー́粉上Č規リ₂井©±½ễńドňŌốộờ胡Šựế号żăʂ玄ćẒěğģḥīオ中²ịœ乙ś慎şţŽūıůŻćčČďĎęěĚĹĽľłŇňřŘšŠťŤůŮŽžăğıİşŞğĞݵćĆčČ𚊎žăĂćčČęěµńőŕřšŠŽž

merges.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f1b317e2025231b7f3c1a8a1baaa1f60f1abcf41000e71cadb55b435cff62ad
+size 155103798

special_tokens_map.json

@@ -0,0 +1 @@
+{}

subtoken_reference.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ebe60b7d10310f37f08504ad458f718fda624eee21a09234f9b9222c6899084
+size 74854818

tokenization_saffu.py

@@ -0,0 +1,1038 @@
+from abc import ABC
+import json, os, re, torch
+from abc import abstractmethod
+from transformers.utils import logging
+from collections import Counter, defaultdict
+from typing import TYPE_CHECKING, List, Optional, Tuple
+from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
+
+logger = logging.get_logger(__name__)
+
+VOCAB_FILES_NAMES = {
+    "subtoken_reference_file": "subtoken_reference.json",
+    "vocab_file": "vocab.json",
+    "merges_file": "merges.json",
+    "chars_file": "chars.txt"
+}
+
+PRETRAINED_VOCAB_FILES_MAP = {
+    "subtoken_reference_file": {
+        "https://huggingface.co/saffu-BBLM10M/resolve/main/subtoken_reference.json",
+        "https://huggingface.co/saffu-BBLM100M/resolve/main/subtoken_reference.json",
+    },
+    "chars_file": {
+        "https://huggingface.co/saffu-BBLM10M/resolve/main/chars.json",
+        "https://huggingface.co/saffu-BBLM100M/resolve/main/chars.json",
+    },
+    "vocab_file": {
+        "saffu-BBLM10M": "https://huggingface.co/saffu-BBLM10M/resolve/main/vocab.json",
+        "saffu-BBLM100M": "https://huggingface.co/saffu-BBLM100M/resolve/main/vocab.json",
+    },
+    "merges_file": {
+        "saffu-BBLM10M": "https://huggingface.co/saffu-BBLM10M/resolve/main/merges.json",
+        "saffu-BBLM100M": "https://huggingface.co/saffu-BBLM100M/resolve/main/merges.json",
+    },
+}
+
+class SAFFUTokenizer(PreTrainedTokenizer):
+    """
+    Construct a SAFFU tokenizer. Based on rule-based pre-tokenization followed by Byte-Pair sub-word chunking.
+    """
+
+    vocab_files_names = VOCAB_FILES_NAMES
+    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    model_input_names = ["input_ids"]
+
+    def __init__(
+        self,
+        vocab_file,
+        subtoken_reference_file,
+        merges_file,
+        chars_file,
+        r = 2,
+        block_size = 100,
+        heads = 2,
+        space = False,
+        pad = "<pad>", 
+        oov = "<oov>", 
+        sod = "<sod>", 
+        eod = "<eod>", 
+        frg = "<frg>",
+        **kwargs,
+    ):
+        super().__init__(
+            **kwargs,
+        )
+    
+        self._r = r
+        self._space = space
+        self._heads = heads
+        self._block_size = block_size
+        self._raw_td = json.load(open(merges_file))
+        self._td = load_td(path = merges_file)
+        self._wordchars = re.sub(" ", "", open(chars_file).read().strip())
+        self._vocabulary = json.loads(open(vocab_file).read())
+        self._index = {self._vocabulary[t]: t for t in self._vocabulary}
+        self._subtoken_reference = json.loads(open(subtoken_reference_file).read())
+        self._pad = pad
+        self._oov = oov
+        self._sod = sod
+        self._eod = eod
+        self._frg = frg
+        self._padding = [self._vocabulary[self._pad]]*self._r
+        self._masking = [self._vocabulary[self._pad]]*self._block_size
+        self._heads_padding = [self._vocabulary[self._pad]]*self._heads
+
+    def save_vocabulary(self, save_directory: str, 
+                        filename_prefix: Optional[str] = None) -> Tuple[str]:
+        if not os.path.isdir(save_directory):
+            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
+            return
+        vocab_file = os.path.join(
+            save_directory, (filename_prefix + "-" if filename_prefix else "") + 
+            VOCAB_FILES_NAMES["vocab_file"]
+        )
+        merge_file = os.path.join(
+            save_directory, (filename_prefix + "-" if filename_prefix else "") +
+            VOCAB_FILES_NAMES["merges_file"]
+        )
+        subtoken_reference_file = os.path.join(
+            save_directory, (filename_prefix + "-" if filename_prefix else "") + 
+            VOCAB_FILES_NAMES["subtoken_reference_file"]
+        )
+        chars_file = os.path.join(
+            save_directory, (filename_prefix + "-" if filename_prefix else "") +
+            VOCAB_FILES_NAMES["chars_file"]
+        )
+        
+        with open(vocab_file, "w", encoding="utf-8") as f:
+            f.write(json.dumps(self._vocabulary, indent=2, sort_keys=True, ensure_ascii=False) + "\n")
+        with open(merge_file, "w", encoding="utf-8") as f:
+            f.write(json.dumps(self._raw_td, indent=2, sort_keys=True, ensure_ascii=False))
+        with open(subtoken_reference_file, "w", encoding="utf-8") as f:
+            f.write(json.dumps(self._subtoken_reference, indent=2, sort_keys=True, ensure_ascii=False))
+        with open(chars_file, "w", encoding="utf-8") as f:
+            f.write(self._wordchars)
+
+        return vocab_file, merge_file, subtoken_reference_file, chars_file
+
+    @property
+    def vocab_size(self):
+        return len(self._vocabulary)
+
+    @staticmethod
+    def word_tokenize(text, wordchars = "a-zA-Z0-9-'"): 
+        return [token for token in re.split("(["+wordchars+"'-]+)", text) if token]
+
+    @staticmethod
+    def stick_spaces(stream):
+        tokens = []
+        for wi, w in enumerate(stream):
+            if not tokens:
+                tokens.append(w)
+            elif w == ' ':
+                if (tokens[-1][-1] != ' ') and (wi != len(stream)-1):
+                    tokens.append(w)
+                else:
+                    tokens[-1] = tokens[-1] + w
+            else:
+                if tokens[-1][-1] == ' ':
+                    if tokens[-1] == ' ':
+                        tokens[-1] = tokens[-1] + w
+                    else:
+                        tokens[-1] = tokens[-1][:-1]#  + w
+                        tokens.append(' ' + w)
+                else:
+                    tokens.append(w)
+        return tokens
+
+    @staticmethod
+    def sentence_tokenize(text, wordchars = "a-zA-Z0-9-'", puncts = ".?!;:\n|"):
+        sentences = []
+        for sentence in re.split("(\s+(?<=["+puncts+"][^"+wordchars+"'-])\s*)", text):
+            if not sentence: continue
+            if not re.search("["+wordchars+"'-]", sentence):
+                if len(sentences):
+                    if sentence[-1] == " ":
+                        if len(sentence) > 1:
+                            sentences[-1] = sentences[-1] + sentence[:-1]
+                            sentences.append(sentence[-1])
+                        else:
+                            sentences.append(sentence)
+                    else:
+                        sentences[-1] = sentences[-1] + sentence
+                else:
+                    sentences.append(sentence)
+            else:
+                if len(sentences):
+                    if len(sentences[-1]) == 1 and sentences[-1] == " ": 
+                        sentences[-1] = sentences[-1] + sentence
+                    else:
+                        sentences.append(sentence)
+                else:
+                    sentences.append(sentence)
+        return sentences
+
+    def bpe_tokenize(self, text): # , td = {}, reference = {}, space = False
+        stream = self._subtoken_reference.get(text, list_tokenize(text, td = self._td))
+        return (list(stream if self._space else self.stick_spaces(stream)))
+    
+    def _tokenize(self, text):
+        """Tokenize a string."""
+        return [sub for s in self.sentence_tokenize(text, wordchars = self._wordchars)
+                for t in (self.word_tokenize(s, wordchars = self._wordchars) if self._space else 
+                          self.stick_spaces(self.word_tokenize(s, wordchars = self._wordchars)))
+                for sub in self.bpe_tokenize(t)]
+        return document
+
+    def preprocess(self, input_ids = []):
+        document  = input_ids + [self._vocabulary[self._eod], self._vocabulary[self._pad]]
+        blocks = []; docsize  = len(document)
+        for bi in range(int(docsize/self._block_size) + 1):
+            start = bi*self._block_size
+            if start > docsize: continue
+            end = min([(bi+1)*self._block_size, docsize])
+            data = [self._vocabulary[self._frg if bi else  self._sod]] + document[start:end]
+            block = (lambda x: x[:3] + [x[3:]])(
+                list(map(list,
+            zip(*[(t, self._padding[:self._r - m] + data[:m] if m - self._r < 0 else data[m - self._r:m],
+                   data[:m] + self._masking[:self._block_size - m] if m < self._block_size else data[:self._block_size],
+                   *(self._heads_padding[:self._heads - m] + data[:m] if m - self._heads < 0 else 
+                     data[m - self._heads:m])[::-1]) # last heads tokens
+                  for m, t in enumerate(data)])
+            )) )
+            blocks.append(block)
+        return blocks
+
+    def _convert_token_to_id(self, t):
+        """Converts a token (str) in an id using the vocab."""
+        return self._vocabulary.get(t, self._vocabulary.get(self._oov))
+
+    def _convert_id_to_token(self, i):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        return self._index.get(i, self._oov)
+
+def load_td(data = None, path = ''):
+    if data is None and path:
+        data = json.load(open(path))
+    td = {}
+    # from Tokenizer
+    td['tok2ind'] = data['tok2ind']
+    td['ind2tok'] = {v: k for k, v in td['tok2ind'].items()}
+    td['action_trace'] = [{'pair': tuple(a[0]), 'type': 'merge' if a[1] else 'split', 
+                           'count': a[2], 'score': a[3]} for a in data['action_trace']]
+    td['tok2acts'] = defaultdict(list)
+    td['pair2merge'] = dict()
+    td['tok2splits'] = defaultdict(list)
+    for aix, a in enumerate(td['action_trace']):
+        if a['type'] =='split':
+            td['tok2acts']["".join(a['pair'])].append(aix)
+            td['tok2splits']["".join(a['pair'])].append(aix)
+        else:
+            td['pair2merge'][tuple(a['pair'])] = aix
+            td['tok2acts'][a['pair'][0]].append(aix)
+            td['tok2acts'][a['pair'][1]].append(aix)
+    td['maxtoklen'] = max([len(t) for t in td['tok2ind']])
+    # from BPE
+    if 'unigraph' in data:
+        td['unigraph'] = Counter(data['unigraph'])
+        td['digraph'] = Counter({(l, r): v for l, r, v in data['digraph']})
+        td['doc_unigraph'] = defaultdict(Counter)
+        for k, v in data['doc_unigraph'].items():
+            td['doc_unigraph'][k] = Counter(v)
+    td['init_method'] = data['init_method']
+    # from HRBPE
+    if 'param_method' in data:
+        td['param_method'] = data['param_method']
+        td['reg_model'] = data['reg_model']
+        td['early_stop'] = data['early_stop']
+    return td
+
+def list_tokenize(text, td = {}):
+    assert td['action_trace'], "Can't tokenize, no trained model!"
+    mock = BPE()
+    mock.init([text], method=td['init_method'], apply=True) # method='char'
+    prev_aix = -1; available_action_indices = []; observed = set(); tokenizing = True
+    while tokenizing:
+        available_action_indices = sorted(list(filter(lambda next_aix: next_aix > prev_aix, available_action_indices)) + 
+                                          [next_aix for next_aix in [aix for tok in mock._unigraph 
+                                                                     for aix in td['tok2splits'][tok] if tok not in observed] + 
+                                           [td['pair2merge'][pair] for pair in mock._digraph
+                                            if pair not in observed and pair in td['pair2merge']]
+                                           if next_aix > prev_aix])
+        observed = observed.union(set(mock._unigraph.keys()).union(set(mock._digraph.keys())))
+        if available_action_indices:
+            aix = available_action_indices[0]
+        else:
+            tokenizing = False
+            break
+        prev_aix = aix
+        action = td['action_trace'][aix]
+        if action['type'] == 'merge':
+            mock.merge(action['pair'])
+        else:
+            mock.split(action['pair'])
+    tks = []
+    for t, idxs in mock._tok_idx.items():
+        for ix in idxs:
+            tks.append((t, ix))
+    tks.sort(key=lambda ti: ti[1])
+    tks, _ = zip(*tks)
+    return tks
+
+# purpose: cast a merge or split action object
+# arguments: see __init__()
+# prereqs: none
+# use methods: none
+# - __init__: 
+# use attributes:
+# - pair: tuple, form of the action
+# - type: str, 'merge' or 'split' describing the type of action
+# - count: int, indicating the number of times an action has been observed (for ranking)
+class Action:
+    # purpose: initialize a merge or split action object
+    # arguments:
+    # - pair: tuple, form of the action
+    # - type: str, 'merge' or 'split' describing the type of action
+    # - count: int, indicating the number of times an action has been observed (for ranking)
+    # prereqs: NA
+    # output: an initialized Action object for merge-split augmented BPE
+    def __init__(self, pair, type='merge', count=-1):
+        self.pair = pair
+        self.type = type
+        self.count = int(count)
+
+# purpose: cast a tokenizer
+# arguments: see __init__()
+# prereqs:
+# use methods: 
+# - __init__: 
+# - save: save a model for later use
+# - load: load a saved model
+# - init: initialize a model
+# - fit: fit a model
+# - tokenize: apply the model's trained tokenizer to a given text
+# - display: plot the rank-frequency distribution of the current segmentation of the system's ingested data, as well as a given frequency model
+# properties: the tokenizer's length is the size of its index
+class Tokenizer(ABC):
+    # purpose: initialize a tokenizer
+    # arguments:
+    # - tok2ind: (optional) dict, used by .load to set the index
+    # prereqs: NA
+    # output: a model shell ready for training or trained parameters
+    def __init__(self, tok2ind=None):
+        if tok2ind is None:
+            self._tok2ind = {}
+        else:
+            self._tok2ind = tok2ind
+        self._ind2tok = {v: k for k, v in self._tok2ind.items()}
+        self._action_trace = []
+        
+    # the tokenizer's length is the size of its index
+    def __len__(self):
+        return len(self._tok2ind)
+    
+    # purpose: add a token to the index
+    # arguments: tok: str, the token to add to the index
+    def add_type(self, tok):
+        if tok not in self._tok2ind:
+            self._tok2ind[tok] = len(self._tok2ind)
+            self._ind2tok[self._tok2ind[tok]] = tok
+            
+    # purpose: delete a type from the index
+    # arguments: tok: str, the token to delete from the index
+    def del_type(self, tok):
+        if tok in self._tok2ind:
+            idx = self._tok2ind[tok]
+            # delete the token from the index and the lookup
+            del self._ind2tok[idx]
+            del self._tok2ind[tok]
+            # shifting down each type that's a larger index
+            # than the one just removed
+            i = idx + 1
+            while i in self._ind2tok:
+                t = self._ind2tok[i]
+                self._tok2ind[t] = i - 1
+                self._ind2tok[i - 1] = t
+                del self._ind2tok[i]
+                
+    # purpose: save a model for later use
+    # arguments:
+    # - path: str, directory location where models are to be saved
+    # - data: dict, with fields 'tok2ind', and 'action_trace', which key a dictionary index mapping the vocabulary, and a list of ranked actions to apply as the tokenizers parameters.
+    # output: saved model parameters in the location defined by path
+    def save(self, path, data=None):
+        if data is None:
+            data = {}
+        data['tok2ind'] = self._tok2ind
+        data['action_trace'] = [[a.pair, 1 if a.type == 'merge' else 0, a.count, a.score if hasattr(a, "score") else None]
+                                for a in self._action_trace]
+        json.dump(data, open(path, 'w+'))
+        
+    # purpose: load a saved model
+    # arguments: path: str, directory location from which the model will be loaded
+    # prereqs: a saved model
+    # output: loaded model parameters for operation of a trained tokenizer
+    def load(self, path):
+        data = json.load(open(path))
+        self._tok2ind = data['tok2ind']
+        self._ind2tok = {v: k for k, v in self._tok2ind.items()}
+        self._action_trace = [ScoredAction(tuple(a[0]), count=a[2], score=a[3], 
+                                           type='merge' if a[1] else 'split') 
+                              for a in data['action_trace']]
+        self._tok2acts = defaultdict(list)
+        self._pair2merge = dict()
+        self._tok2splits = defaultdict(list)
+        for aix, a in enumerate(self._action_trace):
+            if a.type =='split':
+                self._tok2acts["".join(a.pair)].append(aix)
+                self._tok2splits["".join(a.pair)].append(aix)
+            else:
+                self._pair2merge[tuple(a.pair)] = aix
+                self._tok2acts[a.pair[0]].append(aix)
+                self._tok2acts[a.pair[1]].append(aix)
+        self._maxtoklen = max([len(t) for t in self._tok2ind])
+        
+        return data
+
+    # purpose: initialize a model
+    # arguments:
+    # - docs: list (corpus) of strs (documents) to be used for training
+    # prereqs: a selected BPE variant, which is what will inherit Tokenizer
+    @abstractmethod
+    def init(self, docs, seed=None):
+        raise NotImplementedError
+
+    # purpose: fit a model
+    # arguments:
+    # - num_batches: int, indicating the number of batches over which to rank and apply actions
+    # - batch_size: int, indicating the number of actions to rank for each batch
+    # - seed: int, indicating the seed of randomization
+    # prereqs: a selected BPE variant, which is what will inherit Tokenizer
+    # output: a trained model
+    @abstractmethod
+    def fit(self, num_batches, batch_size=1, seed=None):
+        raise NotImplementedError
+        
+    # purpose: tokenize a text and map its tokens to vocabulary indices
+    # arguments: text: str, to be tokenized
+    # prereqs: a trained tokenizer
+    # output: a list of token indices, mapped according to the model's vocabulary
+    def encode(self, text):
+        return self.tokens_to_indices(self.tokenize(text))
+
+    # purpose: apply the model's trained tokenizer to a given text
+    # arguments:
+    # - text: str, target text to be tokenized
+    # - start: 
+    # prereqs: a trained tokenizer with an _action_trace, recording the ranked list of tokenization actions
+    # output: a tuple of strs (tokens), which if joined together form the text
+    def tokenize(self, text, start=-1):
+        assert self._action_trace, "Can't tokenize, no trained model!"
+        return self.apply_action_trace(text)
+        
+    # purpose: update the index of actions according to those which are applicable to the data
+    # arguments:
+    # - available_action_indices: list (actions list) of ints (indices) indicating those that are currently available for non-null operation
+    # - model: a trained model object 
+    # - prev_aix: int, indicating the previous action index that was applied
+    # - observed: set, indicating the set of observed tokens within the current tokenization of the data
+    def update_action_indices(self, available_action_indices, model, prev_aix = -1, observed = set()):
+        available_action_indices = sorted(list(filter(lambda next_aix: next_aix > prev_aix, available_action_indices)) + 
+                                          [next_aix for next_aix in [aix for tok in model._unigraph 
+                                                                     for aix in self._tok2splits[tok] if tok not in observed] + 
+                                                                    [self._pair2merge[pair] for pair in model._digraph
+                                                                     if pair not in observed and pair in self._pair2merge]
+                                           if next_aix > prev_aix])
+        observed = observed.union(set(model._unigraph.keys()).union(set(model._digraph.keys())))
+        return available_action_indices, observed
+
+    # purpose: apply a trained models list of actions to a given text (operate BPE)
+    # arguments:
+    # - text: str, document to be tokenized
+    # prereqs: a trained model
+    # output: tuple (document) of strings (tokens)
+    def apply_action_trace(self, text):
+        mock = BPE()
+        mock.init([text], method=self._init_method, apply=True) # method='char'
+        prev_aix = -1; available_action_indices = []; observed = set(); tokenizing = True
+        while tokenizing:
+            available_action_indices, observed = self.update_action_indices(available_action_indices, mock, 
+                                                                            prev_aix = prev_aix, observed = observed)
+            if available_action_indices:
+                aix = available_action_indices[0]
+            else:
+                tokenizing = False
+                break
+            prev_aix = aix
+            action = self._action_trace[aix]
+            if action.type == 'merge':
+                mock.merge(action.pair)
+            else:
+                mock.split(action.pair)
+        tks = []
+        for t, idxs in mock._tok_idx.items():
+            for ix in idxs:
+                tks.append((t, ix))
+        tks.sort(key=lambda ti: ti[1])
+        tks, _ = zip(*tks)
+        return tks
+    
+    def return_tokenization(self):
+        tks = []
+        for t, idxs in self._tok_idx.items():
+            for ix in idxs:
+                tks.append((t, ix))
+        tks.sort(key=lambda ti: ti[1])
+        tks, _ = zip(*tks)
+        return tks
+
+    # purpose: convert a list of token indices to a str object
+    # arguments: indices: list (tokenized document) of ints (indices) to be mapped to strings and joined
+    # prereqs: a trained model
+    # output: str, representing the underlying form of the list of tokens
+    def decode(self, indices):
+        return ''.join(self.indices_to_tokens(indices))
+
+    # purpose: convert str tokens to a list of int token indices
+    # arguments:
+    # - toks: list (document) of strs (tokens) to be mapped theought the model's index to indices
+    # prereqs: a trained model
+    # output: a list of ints (indices) representing the model's tokenized representation of the document
+    def tokens_to_indices(self, toks):
+        return [self._tok2ind[t] for t in toks]
+
+    # purpose: conver a list of token indices to a list of str objects
+    # arguments: indices: list (tokenized document) of ints (indices) to be mapped to strings and joined
+    # prereqs: a trained model
+    # output: list of strs, representing the tokenized form of the document's tokens
+    def indices_to_tokens(self, indices):
+        return [self._ind2tok[i] for i in indices]
+
+# purpose: cast a BPE-based tokenizer object
+# arguments: see __init__()
+# use methods: 
+# - __init__: intialize a BPE-based tokenizer object
+# - save: (see Tokenizer)
+# - init: (see Tokenizer)
+# - load: (see Tokenizer)
+# use attributes: none
+class BPE(Tokenizer):
+    # purpose: initialize a BPE-based tokenizer object
+    # arguments:
+    # - tok2ind: dict (see Tokenizer)
+    # - covering_vocab: set, indicating a collection of strs that the tokenizer should consider as bounds for the result of all possible actions
+    def __init__(self, tok2ind=None, covering_vocab = set()):
+        # defining a covering vocabulary restricts merge/split-able pathways
+        self._covering_vocab = covering_vocab
+        self._covered = {}
+        self._covering = {}
+        if self._covering_vocab:
+            if tok2ind:
+                tok2ind = {t: i for i, t in enumerate(set(list(tok2ind.keys())+list(self._covering_vocab)))}
+            else:
+                tok2ind = {t: i for i, t in enumerate(self._covering_vocab)}
+        # initialize token index, now that the cover is included
+        super().__init__(tok2ind=tok2ind)
+        # starting and ending points for each token (as a set for constant lookup)
+        self._lefts = {}
+        self._rights = {}
+        # frequency-based information
+        self._unigraph = Counter()
+        self._doc_unigraph = defaultdict(Counter)
+        self._digraph = Counter()
+        # mapping to and from indices
+        self._tok_idx = defaultdict(set)
+        self._pair_idx = defaultdict(set)
+        self._char2docidx = {}        
+
+    # purpose: save a model
+    # arguments:
+    # - path: (see Tokenizer)
+    # - data: (see Tokenizer)
+    def save(self, path, data=None):
+        if data is None:
+            data = {}
+        data['unigraph'] = dict(self._unigraph)
+        data['digraph'] = [[k[0], k[1], v] for k, v in self._digraph.items()]
+        data['doc_unigraph'] = {k: dict(v) for k, v in self._doc_unigraph.items()}
+        data['init_method'] = self._init_method
+        super(BPE, self).save(path, data=data)
+#         json.dump({'docs': self._training_data, 'covering': self._training_covering}, 
+#                   open(re.sub('.json', '-docs.json', path), 'w+'))
+
+    # purpose: load a model
+    # arguments:
+    # - path: (see Tokenizer)
+    def load(self, path):
+        data = super(BPE, self).load(path)
+        self._unigraph = Counter(data['unigraph'])
+        self._digraph = Counter({(l, r): v for l, r, v in data['digraph']})
+        self._doc_unigraph = defaultdict(Counter)
+        for k, v in data['doc_unigraph'].items():
+            self._doc_unigraph[k] = Counter(v)
+        self._init_method = data['init_method']
+        return data
+
+    # purpose: intialize a BPE-based model
+    # arguments:
+    # - docs: list (corpus) of strs (document), containing the data on which the model will be trained
+    # - seed: int, indicating the seed of randomization
+    # - method: str, one from: 'char' (start from characters), 'warm' (start from a space-based segmentation), or 'rand' (start from a random segmentation)
+    # - apply: whether or not this will be an application of the action trace post-training, which just shuts off a progress bar
+    # - covering: list (corpus) of lists (documents) of strs (tokens), representing a collection of token boundaries that must be observed during learning, i.e., restricting the learnable rules.
+    # - action_protect: list of strs, indicating regular expressions of that cannot be included in actions, protecting the model from, e.g., learning to merge known unwanted tokens
+    # prereqs: a corpus of document to either tokenize or initialize for training
+    # output: none, data are ingested and structured for learning or application of a model
+    def init(self, docs, seed=None, method='char', apply=False, covering = [], action_protect = ''):
+        # self._training_data = docs; self._training_covering = covering
+        ##
+        self._doc_counts = Counter(); ds = []; cs = []; doc_index = {}
+        for di, doc in enumerate(docs): # tqdm(list(), desc = "Indexing and counting multiplicity of documents"):
+            if doc not in doc_index:
+                doc_index[doc] = len(ds)
+                ds.append(doc)
+                if covering:
+                    cs.append(covering[di])
+            self._doc_counts[doc_index[doc]] += docs[doc] if type(docs) == Counter else 1
+        docs = ds; covering = cs
+        ##
+        self._init_method = method
+        self._action_protect = action_protect
+        ## guarentees a covering
+        self._covering = {}
+        self._hascover = bool(covering)
+        ix = 0
+        if covering:
+            # assert(len(docs) == len(covering))
+            for doc, segmentation in zip(docs, covering):
+                for s_ix, s in enumerate(segmentation):
+                    for ch in s:
+                        self._covering[ix] = s_ix
+                        ix += 1
+                ix += 1
+        d_ix = 0
+        s_ix = max(self._covering.values()) if self._covering else -1
+        for doc in docs:
+            if d_ix + len(doc) > ix:
+                s_ix += 1
+            for ch in doc:
+                if d_ix > ix:
+                    self._covering[d_ix] = s_ix
+                d_ix += 1
+            d_ix += 1
+        if seed:
+            np.random.seed(seed=seed)
+        offset = 0
+        for doc_idx, doc in enumerate(docs if apply else tqdm(docs, desc=f'Initializing')):
+            stream = self._init_doc(doc, method=method)
+            assert (sum(map(len, stream)) == len(doc))
+            for ix, tok in enumerate(stream):
+                self._unigraph[tok] += self._doc_counts[doc_idx] # 1
+                self._tok_idx[tok].add(offset)
+                for char_idx in range(offset, offset + len(tok)):
+                    self._char2docidx[char_idx] = doc_idx #####################################
+                self._doc_unigraph[doc_idx][tok] += self._doc_counts[doc_idx] # 1
+                tok_pair = (stream[ix - 1], tok) if ix else ('', tok)
+                self._lefts[(offset - len(stream[ix - 1])) if ix else (offset - 1)] = tok_pair
+                self._rights[offset] = tok_pair
+                if ix:
+                    self._digraph[tok_pair] += self._doc_counts[doc_idx] # 1
+                    self._pair_idx[tok_pair].add(offset - len(stream[ix - 1]))
+                offset += len(tok)
+            tok_pair = (tok, '')
+            self._lefts[offset - len(tok)] = tok_pair
+            self._rights[offset] = tok_pair
+            offset += 1
+
+    # purpose: initialize a document for training or tokenization
+    # arguments:
+    # - d: str, representing the underlying data to be used for training, or which will be tokenized
+    # - method: str, indicating method for initialization. one of: 'char' (start from characters), 'warm' (start from a space-based segmentation), or 'rand' (start from a random segmentation)
+    # output: a list of strings, which will serve as starting points for tokenization by operation of actions
+    @staticmethod
+    def _init_doc(d, method='char'):
+        if method == 'char':
+            return d
+        elif method == 'warm':
+            return [token for token in re.split("([a-zA-Z0-9-']+)", d) if token] # tokenize(d)
+        elif method == 'rand':
+            topidx = sorted(set(
+                [0] + sorted(np.random.choice(np.arange(1, len(d)), size=int(len(d) / 2), replace=False)) + [len(d)]))
+            return [d[topidx[idx - 1]:topidx[idx]] for idx in range(1, len(topidx))]
+        else:
+            raise ValueError(f'Unrecognized document pre-processing method: {method}')
+
+    # purpose: determine if a given pair can be merged, based on whether the join of its constituents is a substring of the cover
+    # arguments:
+    # - pair: tuple of strs, indicating two tokens adjacent to one another
+    # output: boolean, indicating whether or not the pair's join would compatible with the cover
+    def under_cover(self, pair): # for span-level covering
+        newtok = "".join(pair)
+        skip_next = False
+        for i in sorted(list(self._pair_idx[pair])):
+            if skip_next:  # handle odd numbers of repeated tokens
+                skip_next = False
+                continue
+            skip_next = True if pair[0] == pair[1] and pair[1] == self._lefts[i + len(pair[0])][1] else False
+            if (i in self._covering) and (i+len(newtok)-1 in self._covering):
+                if self._covering[i] != self._covering[i+len(newtok)-1]:
+                    return False
+            elif (i in self._covering) or ((i+len(newtok)-1) in self._covering):
+                return False                
+        else:
+            return True
+
+    # purpose: determine if a given pair can be split, based on whether breaking its consitituents violates the cover
+    # arguments:
+    # - wpair: tuple of strs whose interaction with the cover will be evaluated 
+    # output: bool, indicating whether or not the split degrades the efficiency of the cover
+    def split_under_cover(self, wpair): # for span-level covering
+        oldtok = "".join(wpair)
+        locations = list(self._tok_idx[oldtok])
+        for i in sorted(locations):
+            if (self._covering[i] != self._covering[i+len(wpair[0])-1] or 
+                self._covering[i+len(wpair[0])] != self._covering[i+len(wpair[0])+len(wpair[1])-1]):
+                return False
+        else:
+            return True
+
+    # purpose: determines if a given str is a token within the covering vocabulary
+    # arguments: newtok: str, to be evaluated for substring status within at least one token contained in the covering vocabulary
+    # output: bool, indicating if the token is a substring one from the covering vocabulary
+    def is_covered(self, newtok): # for vocab-level covering
+        if newtok in self._covered: 
+            return self._covered[newtok]
+        else:
+            for cover_token in self._covering_vocab:
+                if newtok in cover_token:
+                    self._covered[newtok] = True
+                    return self._covered[newtok]
+            else:
+                self._covered[newtok] = False
+                return self._covered[newtok]
+
+    # purpose: determines if token covers a token within a covering vocabulary
+    # arguments: newtok: str, to be evaluated for superstring status over at least one token contained in the covering vocabulary
+    # output: bool, indicating if the token is a superstring of one from the covering vocabulary
+    def is_covering(self, newtok): # for vocab-level covering
+        if newtok in self._covering: 
+            return self._covering[newtok]
+        else:
+            for cover_token in self._covering_vocab:
+                if cover_token in newtok:
+                    self._covering[newtok] = True
+                    return self._covering[newtok]
+            else:
+                self._covering[newtok] = False
+                return self._covering[newtok]
+
+    # purpose: trains an hr-bpe model over the system's current state of ingested data
+    # arguments:
+    # - num_batches: int, indicating the number of iterations of action ranking into test batches that will be operated
+    # - batch_size: int, indicating the number of potentially-optimizing actions to rank per test batch (merge and split, each)
+    # - actions_per_batch: int, indicating the number of optimizing actions to sample and test for inclusion as learned rules, per test batch
+    # - seed: int, for control of randomization sampling
+    # output: NA, method modifies the state of model parameters by learning optimizing actions
+    def fit(self, num_batches, batch_size=1, actions_per_batch=None, seed=None):
+        if seed:
+            np.random.seed(seed=seed)
+
+        if actions_per_batch is None:
+            actions_per_batch = batch_size
+        elif actions_per_batch > batch_size:
+            actions_per_batch = batch_size
+
+        pbar = tqdm(total=self._early_stop, desc = 'Fitting')
+        for batch in range(num_batches): # tqdm(range(num_batches), desc='Fitting'):
+            actions = self.rank_actions(self.get_actions(batch_size, actions_per_batch)) # [:batch_size]
+            for action in actions:
+                vsize = len(self._unigraph)
+                if action.type == 'merge':
+                    ## Different criteria for avoiding merges
+                    newtok = "".join(action.pair)
+                    if self._action_protect:
+                        if re.search("("+"|".join(self._action_protect)+")", newtok): continue
+                    if self._hascover:
+                        if not self.under_cover(action.pair):
+                            continue
+                    if self._covering_vocab:
+                        if (not self.is_covered(newtok)) and (not self.is_covering(newtok)):
+                            continue
+                    self.merge(action.pair)
+                else:
+                    ## Different criteria for avoiding splits
+                    if self._action_protect:
+                        if (re.search("("+"|".join(self._action_protect)+")", action.pair[0]) or
+                            re.search("("+"|".join(self._action_protect)+")", action.pair[1])): 
+                            continue
+                    if self._hascover:
+                        if not self.split_under_cover(action.pair):
+                            continue
+                    if self._covering_vocab:
+                        if (((not self.is_covered(action.pair[0])) and (not self.is_covering(action.pair[0]))) or
+                            ((not self.is_covered(action.pair[1])) and (not self.is_covering(action.pair[1])))):
+                            continue
+                    self.split(action.pair)
+                    
+                self._action_trace.append(action)
+                pbar.update(len(self._unigraph) - vsize)
+                if self.do_break_early() or not actions:
+                    break
+            if self.do_break_early() or not actions:
+                break
+
+        # for k in self._unigraph.keys():
+        for k, v in sorted(self._unigraph.items(), key=lambda kv: kv[1], reverse=True):
+            self.add_type(k)
+
+        self._tok2acts = defaultdict(list)
+        self._pair2merge = dict()
+        self._tok2splits = defaultdict(list)
+        for aix, a in enumerate(self._action_trace):
+            if a.type =='split':
+                self._tok2acts["".join(a.pair)].append(aix)
+                self._tok2splits["".join(a.pair)].append(aix)
+            else:
+                self._pair2merge[tuple(a.pair)] = aix
+                self._tok2acts[a.pair[0]].append(aix)
+                self._tok2acts[a.pair[1]].append(aix)
+        self._maxtoklen = max([len(t) for t in self._tok2ind])
+
+        print(f'Built a vocabulary of {len(self)} types')
+
+    # purpose: concatenate (merge) all adjacent pairs of a given type
+    # arguments:
+    # - pair: tuple of strs, indicating the types of two adjacent tokens
+    # output: an updated representation of the ingested documents, with all pairs of the given type merged
+    def merge(self, pair):
+        newtok = "".join(pair)
+
+        skip_next = False
+        locations = list(self._pair_idx[pair])
+        for i in sorted(locations):
+            if skip_next:  # handle odd numbers of repeated tokens
+                skip_next = False
+                continue
+
+            # gather the instance's neighbors
+            lneighbor = self._rights[i][0]
+            rneighbor = self._lefts[i + len(pair[0])][1]
+            skip_next = True if pair[0] == pair[1] and pair[1] == rneighbor else False
+
+            # delete the entries for this pair in both indices
+            del (self._lefts[i])
+            del (self._rights[i + len(pair[0])])
+
+            # gather the old left and right adjacent pairs
+            lpair = (lneighbor, pair[0])
+            rpair = (pair[1], rneighbor)
+
+            # construct new left and right adjacent pairs
+            newlpair = (lneighbor, newtok)
+            newrpair = (newtok, rneighbor)
+
+            # delete the old left and right pair from both left/right indexings
+            del (self._lefts[i - len(lneighbor) if lneighbor else i - 1])  # lpair
+            del (self._rights[i])  # lpair
+            del (self._lefts[i + len(pair[0])])  # rpair
+            del (self._rights[i + len(newtok)])  # rpair
+
+            # update both left and right indexings with the new left and right pairs
+            self._lefts[i - len(lneighbor) if lneighbor else i - 1] = newlpair
+            self._rights[i] = newlpair
+            self._lefts[i] = newrpair
+            self._rights[i + len(newtok)] = newrpair
+
+            texti = self._char2docidx[i]
+            # weight = self._doc_counts[texti]
+            # only update left co-occurrences if lneighbor is non-empty
+            if lneighbor:  # including deleting the lpair instance from codata
+                self._digraph[newlpair] += self._doc_counts[texti] # 1
+                self._digraph[lpair] -= self._doc_counts[texti] # 1
+                self._pair_idx[newlpair].add(i - len(lneighbor))
+                self._pair_idx[lpair].remove(i - len(lneighbor))
+                if not self._digraph[lpair]:
+                    del (self._digraph[lpair])
+                if not self._pair_idx[lpair]:
+                    del (self._pair_idx[lpair])
+
+            # only update right co-occurrences if rneighbor is non-empty
+            if rneighbor:  # including deleting rpair the instance from codata
+                self._digraph[newrpair] += self._doc_counts[texti] # 1
+                self._digraph[rpair] -= self._doc_counts[texti] # 1
+                self._pair_idx[newrpair].add(i)
+                self._pair_idx[rpair].remove(i + len(pair[0]))
+                if not self._digraph[rpair]:
+                    del (self._digraph[rpair])
+                if not self._pair_idx[rpair]:
+                    del (self._pair_idx[rpair])
+
+            # update unigram frequencies
+            self._unigraph[newtok] += self._doc_counts[texti] # 1
+            self._unigraph[pair[0]] -= self._doc_counts[texti] # 1
+            self._unigraph[pair[1]] -= self._doc_counts[texti] # 1
+            if not self._unigraph[pair[0]]:
+                del (self._unigraph[pair[0]])
+            if not self._unigraph[pair[1]]:
+                del (self._unigraph[pair[1]])
+
+            # texti = self._char2docidx[i] ###############################################
+            self._doc_unigraph[texti][newtok] += self._doc_counts[texti] # 1
+            self._doc_unigraph[texti][pair[0]] -= self._doc_counts[texti] # 1
+            if not self._doc_unigraph[texti][pair[0]]:
+                del (self._doc_unigraph[texti][pair[0]])
+            self._doc_unigraph[texti][pair[1]] -= self._doc_counts[texti] # 1
+            if not self._doc_unigraph[texti][pair[1]]:
+                del (self._doc_unigraph[texti][pair[1]])
+
+            # update the token locations
+            self._tok_idx[newtok].add(i)
+            self._tok_idx[pair[0]].remove(i)
+            self._tok_idx[pair[1]].remove(i + len(pair[0]))
+            if not self._tok_idx[pair[0]]:
+                del (self._tok_idx[pair[0]])
+            if not self._tok_idx[pair[1]]:
+                del (self._tok_idx[pair[1]])
+
+            # delete the pair from the co-occurrence data record
+            self._digraph[pair] -= self._doc_counts[texti] # 1
+            self._pair_idx[pair].remove(i)
+            if not self._pair_idx[pair]:
+                del (self._pair_idx[pair])
+            if not self._digraph[pair]:
+                del (self._digraph[pair])
+
+    # purpose: divide (split) all tokens of a given type into a defined pair of types (wpair)
+    # arguments:
+    # - wpair: tuple of strs, indicating the two adjacent types inwo which the given token is split 
+    # output: an updated representation of the ingested documents, with all instances of the given type split
+    def split(self, wpair):
+        oldtok = "".join(wpair)
+        locations = list(self._tok_idx[oldtok])
+        for i in sorted(locations):
+            # update the left/right and consequential digraph indices
+            # wpair[0] updates
+            lneighbor = self._rights[i][0]
+            rneighbor = self._lefts[i][1]
+            lpair = (lneighbor, oldtok)
+            rpair = (oldtok, rneighbor)
+            newlpair = (lneighbor, wpair[0])
+            newcpair = wpair
+            newrpair = (wpair[1], rneighbor)
+
+            texti = self._char2docidx[i]
+            # weight = self._doc_counts[texti]
+            # cpair
+            self._digraph[newcpair] += self._doc_counts[texti] # 1
+            self._pair_idx[newcpair].add(i)
+            self._lefts[i] = wpair
+            self._rights[i + len(wpair[0])] = wpair
+
+            # lpairs
+            del (self._rights[i])
+            self._rights[i] = newlpair
+            del (self._lefts[i - len(lneighbor) if lneighbor else i - 1])
+            self._lefts[i - len(lneighbor) if lneighbor else i - 1] = newlpair
+            if lneighbor:
+                self._digraph[newlpair] += self._doc_counts[texti] # 1
+                self._digraph[lpair] -= self._doc_counts[texti] # 1
+                self._pair_idx[newlpair].add(i - len(lneighbor))
+                self._pair_idx[lpair].remove(i - len(lneighbor))
+                if not self._digraph[lpair]:
+                    del self._digraph[lpair]
+                if not self._pair_idx[lpair]:
+                    del (self._pair_idx[lpair])
+
+            # rpairs
+            # del(left_indexed_pairs[i]) # technically, this was just overwritten w/wpair and doesn't need deletion
+            self._lefts[i + len(wpair[0])] = newrpair
+            # del(right_indexed_pairs[i+len(oldtok)])
+            self._rights[i + len(oldtok)] = newrpair
+            if rneighbor:
+                self._digraph[newrpair] += self._doc_counts[texti] # 1
+                self._digraph[rpair] -= self._doc_counts[texti] # 1
+                self._pair_idx[newrpair].add(i + len(wpair[0]))
+                self._pair_idx[rpair].remove(i)
+                if not self._digraph[rpair]:
+                    del (self._digraph[rpair])
+                if not self._pair_idx[rpair]:
+                    del (self._pair_idx[rpair])
+
+            # update unigram frequencies
+            self._unigraph[oldtok] -= self._doc_counts[texti] # 1
+            self._unigraph[wpair[0]] += self._doc_counts[texti] # 1
+            self._unigraph[wpair[1]] += self._doc_counts[texti] # 1
+            if not self._unigraph[oldtok]:
+                del self._unigraph[oldtok]
+
+            # update the token locations
+            self._tok_idx[oldtok].remove(i)
+            self._tok_idx[wpair[0]].add(i)
+            self._tok_idx[wpair[1]].add(i + len(wpair[0]))
+            if not self._tok_idx[oldtok]:
+                del (self._tok_idx[oldtok])
+
+            # texti = self._char2docidx[i]#########################################
+            self._doc_unigraph[texti][oldtok] -= self._doc_counts[texti] # 1
+            if not self._doc_unigraph[texti][oldtok]:
+                del (self._doc_unigraph[texti][oldtok])
+            self._doc_unigraph[texti][wpair[0]] += self._doc_counts[texti] # 1
+            self._doc_unigraph[texti][wpair[1]] += self._doc_counts[texti] # 1
+
+    # purpose: return a list of actions, ranked according to the given system settings (see: HRBPE from regularized, GreedyBPE from greedy)
+    # arguments:
+    # - batch_size: int, indicating the number of potentially-optimizing actions to rank per test batch (merge and split, each)
+    # - actions_per_batch: int, indicating the number of optimizing actions to sample and test for inclusion as learned rules, per test batch
+    # output: list of Action objects
+    def get_actions(self, batch_size, actions_per_batch):
+        raise NotImplementedError
+
+    # purpose: rank a list of actions according to the system's given settings (see: HRBPE from regularized, GreedyBPE from greedy)
+    # arguments:
+    # - actions: list of Action objects
+    # output: list of Action objects, ordered by decreasing sorting value, such as count 
+    def rank_actions(self, actions):
+        raise NotImplementedError
+
+    # purpose: halt the given training process according to the system settings (see: HRBPE from regularized, GreedyBPE from greedy)
+    # arguments: NA
+    # output: boolean, indicating whether or not a stopping criterion has been reached
+    def do_break_early(self):
+        return False
+
+# purpose: instantiate a standard bpe model that greedily accepts merges of highest co-frequency
+# arguments: (see __init__ from base.BPE)
+# prereqs: (see base.BPE)
+# use methods: (see base.BPE)
+# use attributes: (see base.BPE)
+class GreedyBPE(BPE):
+    
+    # - tok2ind: (optional) dict, used by .load to set the index
+    # - covering_vocab: set, indicating a collection of strs that the tokenizer should consider as bounds for the result of all possible actions
+    # - early_stop: bool, with True indicating the model should stop early, i.e., once no actions are predicted to optimize the negative log likelihood
+    def __init__(self, tok2ind=None, covering_vocab = set(), early_stop=1_000_000_000):
+        super().__init__(tok2ind=tok2ind, covering_vocab = covering_vocab)
+        self._early_stop = early_stop
+    
+    # purpose: return a list of actions, ranked according to the current count value for each action's pair of tokens
+    # arguments:
+    # - batch_size: int, indicating the number of potentially-optimizing actions to rank per test batch (merge and split, each)
+    # output: list of Action objects
+    def get_actions(self, batch_size, _):
+        return [Action(pair, type='merge', count=cnt) for pair, cnt in self._digraph.most_common(batch_size)]
+
+    # purpose: rank a list of actions according to the system's current count value for each action's pair of tokens
+    # arguments:
+    # - actions: list of Action objects
+    # output: list of Action objects, ordered by decreasing sorting value, such as count 
+    def rank_actions(self, actions):
+        return sorted(actions, reverse=True, key=lambda a: a.count)
+
+    # purpose: halt the given training process when the vocabulary is the desired size (equal to self._early_stop)
+    # arguments: NA
+    # output: boolean, indicating whether or not a stopping criterion has been reached
+    def do_break_early(self):
+        ## the vocabulary size exceeds the self._early_stop (limit) or largest co-frequency pair has count 1
+        return((len(self._unigraph) >= self._early_stop and self._early_stop) or 
+               self.get_actions(1, 1)[0].count == 1)

tokenizer_config.json

@@ -0,0 +1,11 @@
+{
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_saffu.SAFFUTokenizer",
+      null
+    ]
+  },
+  "clean_up_tokenization_spaces": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "SAFFUTokenizer"
+}

vocab.json

@@ -0,0 +1,4085 @@
+{
+  "\t": 88,
+  "\t\"": 1895,
+  " ": 12,
+  "  ": 484,
+  "   ": 507,
+  "    ": 819,
+  "     ": 2176,
+  "       ": 3203,
+  " !": 213,
+  " \"": 40,
+  " #": 1825,
+  " &": 748,
+  " '": 304,
+  " ''": 200,
+  " 'cause": 2101,
+  " 'd": 2881,
+  " 'em": 1230,
+  " 'll": 1497,
+  " 'm": 2010,
+  " 's": 357,
+  " 've": 2664,
+  " (": 85,
+  " (\"": 1339,
+  " ()": 1649,
+  " (;": 2745,
+  " )": 1448,
+  " *": 2870,
+  " ,": 37,
+  " -": 384,
+  " --": 361,
+  " .": 21,
+  " ..": 1613,
+  " ...": 610,
+  " 1": 729,
+  " 10": 903,
+  " 100": 1567,
+  " 11": 1746,
+  " 12": 1289,
+  " 13": 1867,
+  " 14": 1824,
+  " 15": 1246,
+  " 16": 1692,
+  " 17": 1855,
+  " 18": 1550,
+  " 19": 1669,
+  " 1950": 3812,
+  " 1960": 2912,
+  " 1970": 2732,
+  " 1980": 2578,
+  " 1990": 2684,
+  " 2": 697,
+  " 20": 893,
+  " 200": 3468,
+  " 2000": 2672,
+  " 2001": 3739,
+  " 2002": 3931,
+  " 2003": 4057,
+  " 2004": 3510,
+  " 2005": 3611,
+  " 2006": 2675,
+  " 2007": 3070,
+  " 2008": 2840,
+  " 2009": 3198,
+  " 2010": 1094,
+  " 2011": 1694,
+  " 2012": 1917,
+  " 2013": 1938,
+  " 2014": 2901,
+  " 2015": 3476,
+  " 2016": 2040,
+  " 2017": 2032,
+  " 2018": 2916,
+  " 2020": 3504,
+  " 21": 2313,
+  " 22": 2681,
+  " 23": 2899,
+  " 24": 2085,
+  " 25": 1959,
+  " 26": 3096,
+  " 27": 3246,
+  " 28": 3183,
+  " 29": 3564,
+  " 3": 782,
+  " 30": 1373,
+  " 31": 3999,
+  " 4": 1003,
+  " 40": 2550,
+  " 5": 1015,
+  " 50": 2048,
+  " 6": 1305,
+  " 60": 3584,
+  " 7": 1562,
+  " 8": 1484,
+  " 9": 1871,
+  " :": 1366,
+  " ;": 395,
+  " =": 112,
+  " ?": 41,
+  " A": 229,
+  " ALL": 3920,
+  " AND": 1544,
+  " ARE": 3820,
+  " About": 3338,
+  " Academy": 1985,
+  " According": 3164,
+  " Act": 2645,
+  " Adam": 3860,
+  " Africa": 1533,
+  " African": 2389,
+  " After": 829,
+  " Ah": 2985,
+  " Air": 2121,
+  " Airport": 3998,
+  " Alan": 3303,
+  " Alexander": 3756,
+  " Alice": 3494,
+  " All": 905,
+  " Also": 3563,
+  " Although": 3163,
+  " America": 924,
+  " American": 317,
+  " Americans": 2396,
+  " Amy": 3005,
+  " An": 2267,
+  " And": 356,
+  " Andy": 4052,
+  " Angeles": 2250,
+  " Anne": 1828,
+  " Another": 3706,
+  " April": 680,
+  " Are": 1668,
+  " Army": 1515,
+  " Art": 3093,
+  " Arthur": 3458,
+  " Arts": 3497,
+  " As": 734,
+  " Asia": 2755,
+  " Asian": 3239,
+  " Assembly": 3467,
+  " Association": 2143,
+  " At": 781,
+  " Atlantic": 3252,
+  " August": 689,
+  " Aunt": 2626,
+  " Australia": 1393,
+  " Australian": 2093,
+  " Austria": 3763,
+  " Award": 1579,
+  " Awards": 2807,
+  " B": 992,
+  " BBC": 4059,
+  " BC": 3909,
+  " BE": 3032,
+  " Bank": 3665,
+  " Battle": 3424,
+  " Bavaria": 3866,
+  " Bay": 2546,
+  " Because": 2248,
+  " Before": 3442,
+  " Bell": 4051,
+  " Ben": 2303,
+  " Berlin": 3435,
+  " Best": 2291,
+  " Bible": 3427,
+  " Big": 2454,
+  " Bill": 2078,
+  " Billy": 2677,
+  " Black": 2224,
+  " Blue": 3523,
+  " Board": 3451,
+  " Bob": 2808,
+  " Book": 3963,
+  " Boston": 3072,
+  " Both": 3418,
+  " Brazil": 2887,
+  " Brazilian": 3714,
+  " Britain": 2472,
+  " British": 669,
+  " Brown": 2209,
+  " But": 404,
+  " By": 1420,
+  " C": 1011,
+  " California": 1109,
+  " Can": 2705,
+  " Canada": 1536,
+  " Canadian": 2154,
+  " Captain": 1614,
+  " Carolina": 2826,
+  " Catholic": 2252,
+  " Center": 2242,
+  " Central": 1968,
+  " Chairman": 3635,
+  " Championship": 2031,
+  " Championships": 3513,
+  " Charles": 1744,
+  " Charlie": 2909,
+  " Chicago": 1811,
+  " Chief": 2417,
+  " China": 1463,
+  " Chinese": 1596,
+  " Chris": 3193,
+  " Christ": 2237,
+  " Christian": 1781,
+  " Christmas": 1489,
+  " Church": 1329,
+  " City": 764,
+  " Civil": 3438,
+  " Club": 2456,
+  " College": 1407,
+  " Colonel": 2878,
+  " Columbia": 3738,
+  " Come": 1396,
+  " Committee": 2589,
+  " Company": 2264,
+  " Congress": 2375,
+  " Council": 1399,
+  " County": 551,
+  " Court": 1773,
+  " Cup": 1236,
+  " D": 1243,
+  " DNA": 3455,
+  " DO": 3194,
+  " DON'T": 3686,
+  " Dad": 1903,
+  " Daddy": 4053,
+  " Dan": 3001,
+  " Daniel": 3656,
+  " Danny": 3682,
+  " David": 1300,
+  " Day": 2097,
+  " De": 3972,
+  " December": 708,
+  " Democratic": 2524,
+  " Department": 2319,
+  " Dick": 3016,
+  " Did": 2460,
+  " Director": 3644,
+  " District": 1453,
+  " Division": 2478,
+  " Do": 1244,
+  " Doctor": 2604,
+  " Don": 3367,
+  " Don't": 1571,
+  " Dr": 1075,
+  " Duke": 3560,
+  " During": 1665,
+  " Dutch": 2379,
+  " E": 1492,
+  " Each": 3600,
+  " Earth": 1594,
+  " East": 1172,
+  " Eastern": 3087,
+  " Education": 3919,
+  " Edward": 2740,
+  " Egypt": 3122,
+  " Elizabeth": 3328,
+  " Emperor": 3605,
+  " Empire": 2348,
+  " England": 869,
+  " English": 653,
+  " Europe": 1226,
+  " European": 1273,
+  " Even": 3342,
+  " F": 1842,
+  " FOR": 2799,
+  " Father": 2178,
+  " February": 745,
+  " Federal": 3924,
+  " Festival": 3149,
+  " Film": 3503,
+  " First": 1833,
+  " Florida": 2275,
+  " Football": 2969,
+  " For": 811,
+  " Force": 3067,
+  " Forest": 3793,
+  " Fox": 2271,
+  " France": 631,
+  " Francisco": 3322,
+  " Frank": 2103,
+  " French": 760,
+  " Friday": 2748,
+  " From": 1595,
+  " G": 1880,
+  " Games": 2414,
+  " General": 1299,
+  " George": 1141,
+  " Georgia": 3283,
+  " German": 925,
+  " Germany": 1032,
+  " Get": 2554,
+  " Girl": 3882,
+  " Go": 2306,
+  " God": 364,
+  " God's": 3471,
+  " Golden": 3681,
+  " Good": 1433,
+  " Government": 2295,
+  " Governor": 2637,
+  " Grand": 2470,
+  " Great": 1386,
+  " Greek": 2035,
+  " Green": 2096,
+  " Group": 2525,
+  " H": 2006,
+  " HAVE": 3125,
+  " Hall": 1832,
+  " Harry": 2837,
+  " Have": 3052,
+  " He": 106,
+  " He's": 2269,
+  " Health": 3978,
+  " Hello": 2646,
+  " Henry": 1763,
+  " Her": 1320,
+  " Here": 2025,
+  " Hey": 1298,
+  " Hi": 2475,
+  " High": 1802,
+  " Hill": 2479,
+  " His": 568,
+  " Holy": 3382,
+  " Home": 3724,
+  " Hong": 3717,
+  " House": 1010,
+  " How": 999,
+  " However": 970,
+  " I": 19,
+  " I'd": 606,
+  " I'll": 283,
+  " I'm": 124,
+  " I've": 310,
+  " II": 1190,
+  " III": 3062,
+  " IN": 1597,
+  " IS": 2160,
+  " IT": 1592,
+  " If": 809,
+  " Illinois": 2385,
+  " In": 179,
+  " India": 1153,
+  " Indian": 1281,
+  " Institute": 2544,
+  " International": 1466,
+  " Internet": 3913,
+  " Iowa": 2773,
+  " Iran": 3631,
+  " Ireland": 2452,
+  " Irish": 3008,
+  " Is": 1347,
+  " Island": 1656,
+  " Islands": 3670,
+  " Israel": 3176,
+  " It": 119,
+  " It's": 796,
+  " Italian": 1620,
+  " Italy": 1771,
+  " Its": 1460,
+  " J": 1776,
+  " Jack": 1365,
+  " Jackson": 3039,
+  " James": 1270,
+  " Jane": 2907,
+  " January": 598,
+  " Japan": 1025,
+  " Japanese": 1134,
+  " Jerry": 3872,
+  " Jersey": 3158,
+  " Jesus": 1705,
+  " Jewish": 2708,
+  " Jews": 3848,
+  " Jim": 2374,
+  " Jimmy": 3134,
+  " Jo": 2221,
+  " Joe": 1941,
+  " John": 571,
+  " Johnny": 3201,
+  " Johnson": 2993,
+  " Jones": 3251,
+  " Joseph": 2929,
+  " Jr": 3152,
+  " July": 683,
+  " June": 651,
+  " Just": 1853,
+  " Justice": 4025,
+  " K": 2567,
+  " KNOW": 3248,
+  " Kansas": 3056,
+  " Kentucky": 3937,
+  " Kim": 3381,
+  " King": 679,
+  " Kingdom": 1723,
+  " Kong": 3349,
+  " Korea": 2982,
+  " Korean": 3412,
+  " L": 2029,
+  " La": 2817,
+  " Lady": 2298,
+  " Lake": 2346,
+  " Later": 3079,
+  " Latin": 2585,
+  " Law": 3160,
+  " League": 990,
+  " Lee": 2201,
+  " Let": 2488,
+  " Let's": 3050,
+  " Life": 3217,
+  " Like": 2876,
+  " Line": 3770,
+  " Little": 1989,
+  " London": 854,
+  " Long": 3980,
+  " Look": 2927,
+  " Lord": 1005,
+  " Los": 2167,
+  " Louis": 2593,
+  " Love": 3020,
+  " M": 1357,
+  " ME": 2109,
+  " MY": 3018,
+  " Majesty": 2814,
+  " Major": 2644,
+  " Man": 1840,
+  " Many": 1893,
+  " March": 635,
+  " Maria": 3470,
+  " Mark": 2343,
+  " Martin": 2287,
+  " Mary": 1517,
+  " Massachusetts": 3450,
+  " Master": 2377,
+  " Max": 3935,
+  " May": 605,
+  " Me": 2726,
+  " Mexican": 3932,
+  " Mexico": 2060,
+  " Michael": 1464,
+  " Michigan": 3461,
+  " Middle": 3296,
+  " Mike": 2772,
+  " Minister": 1355,
+  " Miss": 890,
+  " Missouri": 3765,
+  " Mom": 2557,
+  " Monday": 3208,
+  " Moscow": 4035,
+  " Most": 2090,
+  " Mother": 2324,
+  " Mount": 3634,
+  " Mr": 292,
+  " Mrs": 655,
+  " Museum": 2719,
+  " Music": 2575,
+  " My": 1206,
+  " N": 2335,
+  " NOT": 3276,
+  " National": 714,
+  " Nations": 3615,
+  " Navy": 3299,
+  " Netherlands": 3454,
+  " New": 332,
+  " News": 4023,
+  " No": 359,
+  " North": 772,
+  " Northern": 2690,
+  " Not": 1565,
+  " Nothing": 3883,
+  " November": 719,
+  " Now": 1294,
+  " O": 1643,
+  " OF": 1052,
+  " OK": 1454,
+  " ON": 2194,
+  " October": 674,
+  " Of": 1618,
+  " Office": 3417,
+  " Oh": 537,
+  " Ohio": 3528,
+  " Okay": 1348,
+  " Old": 1684,
+  " Olympic": 3457,
+  " Olympics": 2482,
+  " On": 704,
+  " One": 894,
+  " Only": 3344,
+  " Open": 3859,
+  " Other": 2792,
+  " Oxford": 4081,
+  " P": 1527,
+  " Pacific": 3282,
+  " Pakistan": 3408,
+  " Paris": 1408,
+  " Park": 1342,
+  " Parliament": 2614,
+  " Party": 1160,
+  " Paul": 1371,
+  " Pennsylvania": 1661,
+  " People": 2422,
+  " Peter": 995,
+  " Philadelphia": 3870,
+  " Please": 3699,
+  " Poland": 3691,
+  " Police": 3790,
+  " Polish": 4034,
+  " President": 865,
+  " Prime": 2453,
+  " Prince": 1675,
+  " Princess": 2545,
+  " Prize": 3324,
+  " Professor": 2627,
+  " Province": 2866,
+  " Queen": 1697,
+  " R": 1591,
+  " Radio": 4039,
+  " Ray": 3415,
+  " Really": 3992,
+  " Records": 3098,
+  " Red": 1921,
+  " Republic": 1800,
+  " Republican": 2897,
+  " Richard": 1854,
+  " Right": 2512,
+  " River": 1114,
+  " Road": 2175,
+  " Robert": 1696,
+  " Rock": 3587,
+  " Roman": 1663,
+  " Rome": 2439,
+  " Rose": 3174,
+  " Royal": 1792,
+  " Russia": 2183,
+  " Russian": 1648,
+  " S": 1356,
+  " Saint": 3264,
+  " Sam": 2583,
+  " San": 1511,
+  " Santa": 3768,
+  " Sarah": 3675,
+  " Saturday": 2715,
+  " School": 962,
+  " Science": 3896,
+  " Scotland": 2528,
+  " Scott": 3063,
+  " Scottish": 3464,
+  " Sea": 2421,
+  " Second": 3323,
+  " Secretary": 2739,
+  " See": 3053,
+  " Senate": 2987,
+  " September": 673,
+  " Service": 3230,
+  " She": 249,
+  " She's": 3747,
+  " Show": 3472,
+  " Simon": 3752,
+  " Since": 2561,
+  " Sir": 1404,
+  " Smith": 2153,
+  " So": 822,
+  " Society": 2640,
+  " Some": 1177,
+  " South": 675,
+  " Southern": 2841,
+  " Soviet": 1873,
+  " Spain": 2120,
+  " Spanish": 1736,
+  " St": 1017,
+  " Star": 3912,
+  " State": 942,
+  " States": 355,
+  " Station": 4069,
+  " Steve": 3273,
+  " Street": 1416,
+  " Summer": 2458,
+  " Sun": 3074,
+  " Sunday": 2122,
+  " Super": 4029,
+  " Supreme": 3814,
+  " Susan": 4072,
+  " Sweden": 2742,
+  " Swedish": 3720,
+  " Switzerland": 2785,
+  " T": 1577,
+  " THAT": 1974,
+  " THE": 585,
+  " THIS": 2898,
+  " TO": 908,
+  " TV": 1322,
+  " Take": 3842,
+  " Taylor": 3960,
+  " Texas": 1644,
+  " Thank": 1847,
+  " Thanks": 3490,
+  " That": 870,
+  " That's": 1225,
+  " The": 71,
+  " Their": 2255,
+  " Then": 891,
+  " There": 542,
+  " There's": 3740,
+  " These": 1018,
+  " They": 288,
+  " This": 290,
+  " Thomas": 2012,
+  " Three": 3129,
+  " Time": 3254,
+  " Times": 3407,
+  " To": 1120,
+  " Tokyo": 4024,
+  " Tom": 1581,
+  " Tommy": 3505,
+  " Tony": 3105,
+  " Town": 3446,
+  " Township": 3669,
+  " Two": 2207,
+  " U": 712,
+  " UK": 2390,
+  " US": 1303,
+  " Uncle": 2796,
+  " Union": 1430,
+  " United": 300,
+  " University": 580,
+  " V": 2195,
+  " Valley": 2860,
+  " Victoria": 3716,
+  " Virginia": 2017,
+  " W": 2457,
+  " WAS": 2936,
+  " WE": 3818,
+  " WHAT": 3888,
+  " WITH": 3520,
+  " Wales": 2945,
+  " Walter": 3969,
+  " War": 802,
+  " Washington": 1490,
+  " We": 686,
+  " We're": 3292,
+  " Well": 998,
+  " West": 996,
+  " Western": 2116,
+  " What": 350,
+  " What's": 1817,
+  " When": 713,
+  " Where": 1667,
+  " While": 2532,
+  " White": 1877,
+  " Who": 1674,
+  " Why": 1069,
+  " Will": 2523,
+  " William": 1429,
+  " Williams": 3828,
+  " With": 1431,
+  " Women's": 3399,
+  " World": 599,
+  " X": 2398,
+  " YOU": 815,
+  " YOUR": 3206,
+  " Yeah": 619,
+  " Year": 3127,
+  " Yes": 583,
+  " York": 593,
+  " You": 293,
+  " You're": 1500,
+  " Young": 2698,
+  " Your": 1520,
+  " Zealand": 2896,
+  " [": 535,
+  " ]": 724,
+  " _": 779,
+  " `": 1058,
+  " ``": 800,
+  " a": 11,
+  " ability": 2413,
+  " able": 499,
+  " about": 62,
+  " above": 935,
+  " absolutely": 1801,
+  " accept": 1754,
+  " accepted": 2502,
+  " access": 2041,
+  " accident": 1757,
+  " according": 1866,
+  " account": 1627,
+  " acid": 3757,
+  " acquired": 4027,
+  " across": 716,
+  " act": 1186,
+  " acting": 2105,
+  " action": 1248,
+  " actions": 3250,
+  " active": 2106,
+  " activities": 2688,
+  " activity": 2560,
+  " actor": 1436,
+  " actress": 1981,
+  " acts": 3509,
+  " actual": 2279,
+  " actually": 331,
+  " add": 1729,
+  " added": 1284,
+  " addition": 1970,
+  " additional": 3172,
+  " address": 2054,
+  " administration": 3893,
+  " admit": 2911,
+  " adopted": 3365,
+  " adult": 3270,
+  " advance": 3463,
+  " advanced": 3484,
+  " advantage": 2813,
+  " advice": 2443,
+  " affair": 3704,
+  " affairs": 3811,
+  " affected": 3569,
+  " afford": 2904,
+  " afraid": 818,
+  " after": 159,
+  " afternoon": 1421,
+  " afterwards": 2844,
+  " again": 185,
+  " against": 333,
+  " age": 557,
+  " aged": 1646,
+  " agent": 2703,
+  " ago": 545,
+  " agree": 1569,
+  " agreed": 1882,
+  " agreement": 3113,
+  " ah": 2877,
+  " ahead": 1022,
+  " aid": 3453,
+  " ain't": 1008,
+  " air": 707,
+  " aircraft": 2777,
+  " airport": 3391,
+  " album": 699,
+  " albums": 3642,
+  " alive": 1119,
+  " all": 50,
+  " allow": 1603,
+  " allowed": 1163,
+  " allows": 3258,
+  " almost": 572,
+  " alone": 597,
+  " along": 480,
+  " already": 440,
+  " alright": 1170,
+  " also": 138,
+  " alternative": 4080,
+  " although": 1224,
+  " always": 250,
+  " am": 207,
+  " amazing": 1971,
+  " among": 813,
+  " amount": 1296,
+  " an": 60,
+  " analysis": 3284,
+  " ancient": 1745,
+  " and": 10,
+  " anger": 3774,
+  " angle": 3556,
+  " angry": 1688,
+  " animal": 1633,
+  " animals": 1183,
+  " announced": 1390,
+  " annual": 3429,
+  " another": 248,
+  " answer": 794,
+  " answered": 1227,
+  " answers": 3650,
+  " anxious": 3989,
+  " any": 153,
+  " anybody": 1209,
+  " anymore": 1140,
+  " anyone": 803,
+  " anything": 256,
+  " anyway": 1001,
+  " anywhere": 1770,
+  " apart": 1891,
+  " apartment": 2650,
+  " apparently": 3107,
+  " appear": 2046,
+  " appearance": 2320,
+  " appeared": 1097,
+  " appears": 2261,
+  " apple": 3268,
+  " application": 3347,
+  " applied": 3375,
+  " apply": 3364,
+  " appointed": 2145,
+  " appreciate": 3034,
+  " approach": 2562,
+  " appropriate": 3910,
+  " approximately": 3558,
+  " are": 33,
+  " area": 473,
+  " areas": 1256,
+  " aren't": 986,
+  " argument": 3097,
+  " arm": 1462,
+  " armed": 3522,
+  " arms": 1260,
+  " army": 1287,
+  " around": 242,
+  " arranged": 3721,
+  " arrest": 2863,
+  " arrested": 2651,
+  " arrive": 4079,
+  " arrived": 1632,
+  " art": 1068,
+  " article": 2962,
+  " artist": 2067,
+  " artists": 2928,
+  " arts": 3668,
+  " as": 32,
+  " ashamed": 3876,
+  " aside": 3306,
+  " ask": 427,
+  " asked": 465,
+  " asking": 1223,
+  " asleep": 2216,
+  " ass": 1718,
+  " assistant": 3232,
+  " associated": 2989,
+  " association": 3962,
+  " assume": 3491,
+  " at": 36,
+  " ate": 2830,
+  " atmosphere": 3685,
+  " attached": 4005,
+  " attack": 1043,
+  " attacked": 2856,
+  " attacks": 3352,
+  " attempt": 2359,
+  " attend": 3868,
+  " attended": 2915,
+  " attention": 1367,
+  " attitude": 3487,
+  " audience": 2660,
+  " aunt": 3671,
+  " author": 2336,
+  " authorities": 3392,
+  " authority": 2556,
+  " available": 1456,
+  " average": 1474,
+  " avoid": 2770,
+  " awake": 3880,
+  " award": 2798,
+  " awarded": 2485,
+  " awards": 3797,
+  " aware": 2341,
+  " away": 199,
+  " awful": 2308,
+  " b": 3742,
+  " babies": 3199,
+  " baby": 547,
+  " back": 121,
+  " background": 2975,
+  " bad": 417,
+  " badly": 3495,
+  " bag": 1512,
+  " balance": 3516,
+  " ball": 1045,
+  " balls": 3552,
+  " band": 850,
+  " bank": 1481,
+  " banks": 3713,
+  " bar": 1975,
+  " base": 1722,
+  " baseball": 3662,
+  " based": 795,
+  " basic": 2615,
+  " basically": 2066,
+  " basis": 2431,
+  " basketball": 3286,
+  " bastard": 3255,
+  " bath": 3180,
+  " bathroom": 3515,
+  " battle": 1537,
+  " be": 30,
+  " beach": 2631,
+  " bear": 1168,
+  " beat": 1368,
+  " beating": 3841,
+  " beautiful": 663,
+  " beauty": 2068,
+  " became": 349,
+  " because": 147,
+  " become": 550,
+  " becomes": 1844,
+  " becoming": 2053,
+  " bed": 720,
+  " bedroom": 3723,
+  " been": 90,
+  " beer": 3110,
+  " before": 170,
+  " beg": 3397,
+  " began": 475,
+  " begin": 1507,
+  " beginning": 1051,
+  " begins": 2890,
+  " begun": 3832,
+  " behavior": 3337,
+  " behind": 611,
+  " being": 215,
+  " beings": 4068,
+  " belief": 3799,
+  " believe": 379,
+  " believed": 1604,
+  " bell": 3555,
+  " belong": 2734,
+  " belongs": 3499,
+  " below": 1412,
+  " beneath": 3318,
+  " benefit": 3051,
+  " beside": 2605,
+  " besides": 3869,
+  " best": 320,
+  " bet": 1615,
+  " better": 259,
+  " between": 269,
+  " beyond": 1645,
+  " big": 277,
+  " bigger": 1797,
+  " biggest": 2055,
+  " bill": 2648,
+  " billion": 2750,
+  " bird": 1470,
+  " birds": 1815,
+  " birth": 1920,
+  " birthday": 1477,
+  " bit": 341,
+  " bitch": 2164,
+  " bite": 2976,
+  " bits": 3530,
+  " black": 698,
+  " blame": 2810,
+  " blind": 2761,
+  " block": 2387,
+  " blocks": 3440,
+  " blood": 849,
+  " bloody": 2049,
+  " blow": 1870,
+  " blue": 958,
+  " board": 1417,
+  " boat": 1370,
+  " boats": 4017,
+  " bodies": 2489,
+  " body": 554,
+  " bomb": 3173,
+  " bone": 3734,
+  " bones": 3336,
+  " book": 462,
+  " books": 948,
+  " border": 2610,
+  " born": 377,
+  " boss": 1904,
+  " both": 308,
+  " bother": 2804,
+  " bottle": 2434,
+  " bottom": 1458,
+  " bought": 1208,
+  " bound": 2940,
+  " bow": 3796,
+  " box": 959,
+  " boy": 435,
+  " boyfriend": 3290,
+  " boys": 866,
+  " brain": 1415,
+  " branch": 2625,
+  " branches": 3703,
+  " brand": 3705,
+  " brave": 3144,
+  " bread": 2198,
+  " break": 888,
+  " breakfast": 2003,
+  " breaking": 2868,
+  " breath": 2234,
+  " bridge": 1788,
+  " brief": 3557,
+  " bright": 2113,
+  " brilliant": 3380,
+  " bring": 573,
+  " bringing": 2508,
+  " brings": 3136,
+  " broad": 3295,
+  " broadcast": 3602,
+  " broke": 1249,
+  " broken": 1382,
+  " brother": 632,
+  " brothers": 1930,
+  " brought": 603,
+  " brown": 2013,
+  " buddy": 3204,
+  " budget": 3215,
+  " build": 1321,
+  " building": 775,
+  " buildings": 2193,
+  " built": 770,
+  " bullet": 4011,
+  " bunch": 2730,
+  " buried": 2301,
+  " burn": 3021,
+  " burned": 3465,
+  " burning": 3159,
+  " bus": 1799,
+  " business": 559,
+  " busy": 1501,
+  " but": 58,
+  " butter": 3874,
+  " button": 3269,
+  " buy": 834,
+  " buying": 3538,
+  " by": 45,
+  " bye": 2490,
+  " c": 3479,
+  " ca": 3195,
+  " cake": 2437,
+  " call": 297,
+  " called": 220,
+  " calling": 1410,
+  " calls": 1732,
+  " calm": 2404,
+  " came": 212,
+  " camera": 2199,
+  " camp": 1913,
+  " campaign": 2169,
+  " can": 64,
+  " can't": 166,
+  " cancer": 1898,
+  " candidate": 3500,
+  " cannot": 798,
+  " cap": 4042,
+  " capable": 3534,
+  " capacity": 3591,
+  " capital": 1149,
+  " captain": 2084,
+  " captured": 3325,
+  " car": 393,
+  " carbon": 3609,
+  " card": 1786,
+  " cards": 2857,
+  " care": 432,
+  " career": 743,
+  " careful": 1523,
+  " carefully": 2579,
+  " carried": 1440,
+  " carry": 1293,
+  " carrying": 2718,
+  " cars": 1759,
+  " case": 464,
+  " cases": 1516,
+  " cash": 2596,
+  " cast": 2236,
+  " castle": 2573,
+  " cat": 1402,
+  " catch": 1217,
+  " cats": 3950,
+  " caught": 1136,
+  " cause": 1080,
+  " caused": 1319,
+  " causes": 2769,
+  " cell": 1841,
+  " cells": 2408,
+  " census": 1686,
+  " center": 1491,
+  " central": 1918,
+  " centre": 2080,
+  " centuries": 3676,
+  " century": 827,
+  " ceremony": 3881,
+  " certain": 799,
+  " certainly": 1152,
+  " chain": 2843,
+  " chair": 1387,
+  " chairman": 3908,
+  " challenge": 2913,
+  " champion": 3257,
+  " championship": 3483,
+  " chance": 807,
+  " change": 513,
+  " changed": 853,
+  " changes": 1447,
+  " changing": 2658,
+  " channel": 2803,
+  " character": 1124,
+  " characters": 2082,
+  " charge": 1372,
+  " charged": 3847,
+  " charges": 3546,
+  " chart": 3648,
+  " cheap": 3537,
+  " check": 1061,
+  " checked": 3710,
+  " cheese": 2926,
+  " chemical": 3116,
+  " chest": 3151,
+  " chicken": 2629,
+  " chief": 1901,
+  " child": 579,
+  " childhood": 3798,
+  " children": 409,
+  " children's": 3918,
+  " chocolate": 3680,
+  " choice": 1413,
+  " choose": 1751,
+  " chose": 2864,
+  " chosen": 2670,
+  " church": 1020,
+  " circle": 2697,
+  " circumstances": 3131,
+  " cities": 1834,
+  " citizens": 3396,
+  " city": 338,
+  " civil": 2367,
+  " claim": 2552,
+  " claimed": 3214,
+  " claims": 3834,
+  " class": 993,
+  " classes": 2827,
+  " clean": 1262,
+  " clear": 863,
+  " clearly": 2134,
+  " clever": 3378,
+  " client": 3373,
+  " climate": 2972,
+  " climb": 3971,
+  " clock": 3459,
+  " close": 627,
+  " closed": 1401,
+  " closely": 3265,
+  " closer": 2484,
+  " clothes": 1292,
+  " club": 889,
+  " clubs": 3954,
+  " coach": 1836,
+  " coast": 2249,
+  " coat": 2382,
+  " code": 1957,
+  " coffee": 1493,
+  " cold": 910,
+  " collection": 2378,
+  " college": 1314,
+  " color": 1282,
+  " colors": 3387,
+  " colour": 3084,
+  " combination": 3983,
+  " combined": 3968,
+  " come": 134,
+  " comedy": 2847,
+  " comes": 530,
+  " comfort": 3708,
+  " comfortable": 2656,
+  " coming": 360,
+  " command": 2328,
+  " commercial": 2954,
+  " committed": 3133,
+  " committee": 3038,
+  " common": 879,
+  " commonly": 3223,
+  " commune": 1054,
+  " communication": 3826,
+  " communities": 3846,
+  " community": 1088,
+  " companies": 1960,
+  " company": 630,
+  " compared": 3061,
+  " competed": 3242,
+  " competition": 1937,
+  " complete": 1552,
+  " completed": 2400,
+  " completely": 1297,
+  " complex": 2463,
+  " complicated": 3437,
+  " composed": 3329,
+  " computer": 1361,
+  " computers": 3976,
+  " concept": 3154,
+  " concern": 3361,
+  " concerned": 2185,
+  " concert": 3410,
+  " condition": 2034,
+  " conditions": 2023,
+  " conference": 3186,
+  " confidence": 3782,
+  " confirmed": 3878,
+  " conflict": 3817,
+  " confused": 3806,
+  " connected": 2584,
+  " connection": 2766,
+  " consider": 2065,
+  " considered": 1238,
+  " consists": 3460,
+  " constant": 3175,
+  " construction": 2330,
+  " contact": 1979,
+  " contains": 2531,
+  " content": 2551,
+  " continue": 1628,
+  " continued": 1332,
+  " continues": 2922,
+  " contract": 1997,
+  " control": 857,
+  " conversation": 2345,
+  " convinced": 3993,
+  " cook": 2590,
+  " cooking": 3654,
+  " cool": 1286,
+  " cop": 3852,
+  " copies": 3359,
+  " cops": 3927,
+  " copy": 2680,
+  " corner": 1605,
+  " correct": 2449,
+  " cos": 1060,
+  " cost": 1234,
+  " costs": 2793,
+  " could": 122,
+  " couldn't": 676,
+  " council": 2566,
+  " count": 2011,
+  " countries": 1077,
+  " country": 504,
+  " county": 1029,
+  " couple": 821,
+  " courage": 3393,
+  " course": 299,
+  " court": 1210,
+  " cousin": 2984,
+  " cover": 1335,
+  " covered": 1651,
+  " covers": 3755,
+  " cow": 3060,
+  " crap": 3730,
+  " crash": 3639,
+  " crazy": 1040,
+  " cream": 2819,
+  " create": 1503,
+  " created": 901,
+  " creating": 3482,
+  " creation": 3694,
+  " creature": 3153,
+  " credit": 2654,
+  " crew": 2309,
+  " cried": 1126,
+  " crime": 1657,
+  " criminal": 3049,
+  " crisis": 3991,
+  " critical": 3319,
+  " cross": 1856,
+  " crossed": 3647,
+  " crowd": 2908,
+  " crown": 4046,
+  " cry": 1630,
+  " crying": 2222,
+  " cultural": 3475,
+  " culture": 2005,
+  " cup": 1954,
+  " curious": 3300,
+  " current": 1280,
+  " currently": 1902,
+  " customers": 3940,
+  " cut": 732,
+  " cute": 3041,
+  " cutting": 3803,
+  " d": 3333,
+  " da": 3111,
+  " dad": 1039,
+  " daddy": 1590,
+  " daily": 2957,
+  " damage": 2294,
+  " damn": 1859,
+  " dance": 1221,
+  " dancing": 2711,
+  " danger": 2157,
+  " dangerous": 1711,
+  " dare": 2138,
+  " dark": 989,
+  " darkness": 3238,
+  " darling": 2538,
+  " data": 1264,
+  " date": 1233,
+  " dates": 3986,
+  " dating": 3975,
+  " daughter": 693,
+  " daughters": 3974,
+  " day": 188,
+  " days": 388,
+  " de": 613,
+  " dead": 498,
+  " deal": 735,
+  " dealing": 3376,
+  " dear": 727,
+  " death": 472,
+  " debt": 3745,
+  " debut": 2380,
+  " decide": 2086,
+  " decided": 980,
+  " decision": 1513,
+  " decisions": 3758,
+  " deck": 4047,
+  " declared": 2495,
+  " deep": 1232,
+  " deeply": 3955,
+  " defeat": 3394,
+  " defeated": 2653,
+  " defense": 3957,
+  " defined": 3894,
+  " definitely": 1980,
+  " degree": 1784,
+  " degrees": 2977,
+  " demand": 3140,
+  " department": 872,
+  " depends": 3196,
+  " describe": 3237,
+  " described": 1673,
+  " desert": 3664,
+  " deserve": 3624,
+  " design": 1543,
+  " designed": 1933,
+  " desire": 2574,
+  " desk": 3493,
+  " despite": 3593,
+  " destroy": 2728,
+  " destroyed": 2200,
+  " detail": 3576,
+  " details": 2800,
+  " determined": 2930,
+  " develop": 3030,
+  " developed": 1443,
+  " development": 1278,
+  " device": 3425,
+  " devil": 3813,
+  " did": 108,
+  " didn't": 180,
+  " die": 710,
+  " died": 336,
+  " difference": 1330,
+  " different": 302,
+  " difficult": 1087,
+  " difficulty": 3831,
+  " dinner": 1073,
+  " direct": 2355,
+  " directed": 1313,
+  " direction": 1747,
+  " directly": 2110,
+  " director": 1485,
+  " dirty": 2371,
+  " disappeared": 2903,
+  " discovered": 1623,
+  " discuss": 3103,
+  " discussion": 3762,
+  " disease": 1805,
+  " display": 3772,
+  " distance": 1608,
+  " distributed": 3386,
+  " distribution": 3619,
+  " district": 1012,
+  " districts": 3660,
+  " divided": 2064,
+  " division": 2263,
+  " do": 42,
+  " doctor": 1062,
+  " doctors": 3512,
+  " does": 243,
+  " doesn't": 325,
+  " dog": 883,
+  " dogs": 2102,
+  " doing": 204,
+  " dollars": 1780,
+  " don": 2419,
+  " don't": 70,
+  " done": 257,
+  " door": 490,
+  " doors": 2751,
+  " double": 1988,
+  " doubt": 1525,
+  " down": 130,
+  " downstairs": 4082,
+  " dozen": 3616,
+  " drama": 2388,
+  " draw": 1405,
+  " drawing": 2780,
+  " drawn": 2924,
+  " dream": 1272,
+  " dreams": 2285,
+  " dress": 1635,
+  " dressed": 2515,
+  " drew": 2293,
+  " drink": 840,
+  " drinking": 2591,
+  " drive": 1133,
+  " driven": 3884,
+  " driver": 2622,
+  " driving": 1863,
+  " drop": 1621,
+  " dropped": 1765,
+  " drove": 3043,
+  " drug": 2368,
+  " drugs": 2429,
+  " drunk": 2569,
+  " dry": 1838,
+  " duck": 3578,
+  " dude": 3733,
+  " due": 966,
+  " during": 376,
+  " dust": 3589,
+  " duty": 2028,
+  " dying": 2265,
+  " e": 2665,
+  " each": 312,
+  " ear": 2720,
+  " earlier": 1476,
+  " early": 565,
+  " earn": 4045,
+  " earned": 3092,
+  " ears": 2163,
+  " earth": 1259,
+  " easier": 2203,
+  " easily": 1719,
+  " east": 1341,
+  " eastern": 2905,
+  " easy": 808,
+  " eat": 527,
+  " eaten": 3191,
+  " eating": 1578,
+  " economic": 2340,
+  " economy": 2892,
+  " edge": 2061,
+  " edition": 3737,
+  " editor": 3690,
+  " education": 1309,
+  " effect": 1423,
+  " effective": 3434,
+  " effects": 2491,
+  " effort": 2481,
+  " efforts": 3320,
+  " egg": 2854,
+  " eggs": 2117,
+  " eh": 2714,
+  " eight": 843,
+  " eighty": 2971,
+  " either": 643,
+  " elected": 1638,
+  " election": 1336,
+  " elections": 3462,
+  " electric": 3165,
+  " element": 3432,
+  " elements": 2669,
+  " eleven": 2992,
+  " else": 363,
+  " emergency": 3307,
+  " empty": 2083,
+  " end": 309,
+  " ended": 1444,
+  " ending": 3897,
+  " ends": 2307,
+  " enemies": 3389,
+  " enemy": 1789,
+  " energy": 1132,
+  " engaged": 3014,
+  " engine": 2070,
+  " engineer": 4007,
+  " engineering": 3990,
+  " enjoy": 1909,
+  " enjoyed": 3353,
+  " enough": 330,
+  " enter": 2342,
+  " entered": 1750,
+  " entire": 1242,
+  " entirely": 2520,
+  " entrance": 3736,
+  " environment": 2461,
+  " episode": 2220,
+  " episodes": 3939,
+  " equal": 1219,
+  " equipment": 3130,
+  " er": 193,
+  " era": 3637,
+  " erm": 305,
+  " escape": 1864,
+  " escaped": 3776,
+  " especially": 1182,
+  " established": 1514,
+  " estate": 3355,
+  " etc": 2764,
+  " even": 190,
+  " evening": 1028,
+  " event": 1283,
+  " events": 1553,
+  " eventually": 1829,
+  " ever": 289,
+  " every": 284,
+  " everybody": 983,
+  " everyone": 878,
+  " everything": 370,
+  " everywhere": 2339,
+  " evidence": 1334,
+  " evil": 2038,
+  " exact": 3327,
+  " exactly": 832,
+  " example": 649,
+  " examples": 3317,
+  " excellent": 3109,
+  " except": 1395,
+  " exchange": 3188,
+  " excited": 2689,
+  " exciting": 3632,
+  " exclaimed": 3916,
+  " excuse": 2415,
+  " executive": 3767,
+  " exercise": 3362,
+  " exist": 2366,
+  " existence": 2961,
+  " existing": 3949,
+  " expect": 1449,
+  " expected": 1905,
+  " expensive": 2791,
+  " experience": 1154,
+  " experiment": 3946,
+  " explain": 1482,
+  " explained": 3162,
+  " express": 4038,
+  " expression": 2376,
+  " extended": 3688,
+  " extent": 3925,
+  " extra": 2002,
+  " extremely": 2674,
+  " eye": 1113,
+  " eyes": 401,
+  " f": 3141,
+  " face": 398,
+  " faces": 2673,
+  " fact": 536,
+  " factor": 3851,
+  " factory": 3449,
+  " facts": 3719,
+  " fail": 3856,
+  " failed": 2131,
+  " failure": 2657,
+  " fair": 1409,
+  " fairly": 2812,
+  " faith": 2210,
+  " fall": 973,
+  " fallen": 3010,
+  " falling": 2564,
+  " falls": 3298,
+  " false": 3210,
+  " familiar": 2965,
+  " families": 1822,
+  " family": 328,
+  " famous": 1038,
+  " fan": 3623,
+  " fancy": 3083,
+  " fans": 3809,
+  " far": 438,
+  " farm": 2442,
+  " fashion": 2828,
+  " fast": 984,
+  " faster": 2572,
+  " fat": 2170,
+  " fate": 3218,
+  " father": 367,
+  " father's": 2692,
+  " fault": 1702,
+  " favor": 2763,
+  " favorite": 2420,
+  " favour": 3901,
+  " fear": 1269,
+  " feature": 2955,
+  " featured": 3042,
+  " features": 1977,
+  " federal": 2685,
+  " feed": 2504,
+  " feel": 334,
+  " feeling": 949,
+  " feelings": 2329,
+  " feels": 2317,
+  " feet": 765,
+  " fell": 916,
+  " fellow": 1539,
+  " felt": 709,
+  " female": 1699,
+  " festival": 3914,
+  " few": 368,
+  " fiction": 3895,
+  " field": 1036,
+  " fields": 2721,
+  " fifteen": 2594,
+  " fifth": 2842,
+  " fifty": 1469,
+  " fight": 826,
+  " fighting": 1659,
+  " figure": 1013,
+  " figured": 3722,
+  " figures": 2636,
+  " file": 2459,
+  " fill": 2576,
+  " filled": 2000,
+  " film": 751,
+  " films": 2746,
+  " final": 1027,
+  " finally": 1161,
+  " financial": 2516,
+  " find": 223,
+  " finding": 2402,
+  " finds": 2849,
+  " fine": 540,
+  " finger": 2416,
+  " fingers": 2352,
+  " finish": 1391,
+  " finished": 955,
+  " fire": 662,
+  " fired": 2542,
+  " firm": 2606,
+  " first": 126,
+  " fish": 1090,
+  " fishing": 3388,
+  " fit": 1538,
+  " five": 369,
+  " fix": 1878,
+  " fixed": 2321,
+  " flag": 3377,
+  " flat": 2129,
+  " flew": 2696,
+  " flight": 2204,
+  " floor": 1078,
+  " flow": 3403,
+  " flower": 3065,
+  " flowers": 1588,
+  " fly": 1575,
+  " flying": 2177,
+  " focus": 2505,
+  " folks": 2941,
+  " follow": 1185,
+  " followed": 1131,
+  " following": 741,
+  " follows": 3022,
+  " food": 667,
+  " fool": 2243,
+  " foot": 1267,
+  " football": 805,
+  " for": 22,
+  " force": 1074,
+  " forced": 1862,
+  " forces": 1690,
+  " foreign": 2350,
+  " forest": 1810,
+  " forever": 2152,
+  " forget": 974,
+  " forgive": 2638,
+  " forgot": 1774,
+  " forgotten": 2468,
+  " form": 701,
+  " formed": 1394,
+  " former": 752,
+  " forms": 1987,
+  " formula": 1671,
+  " forth": 2107,
+  " fortune": 3536,
+  " forty": 2128,
+  " forward": 1099,
+  " fought": 2418,
+  " found": 240,
+  " founded": 1398,
+  " four": 322,
+  " fourth": 1761,
+  " frame": 3871,
+  " free": 742,
+  " freedom": 2506,
+  " frequently": 4031,
+  " fresh": 2056,
+  " friend": 483,
+  " friendly": 3259,
+  " friends": 581,
+  " friendship": 3938,
+  " frightened": 3294,
+  " frog": 2192,
+  " from": 44,
+  " front": 688,
+  " fruit": 2836,
+  " fuck": 1337,
+  " fucking": 927,
+  " fuel": 3715,
+  " full": 604,
+  " fully": 2630,
+  " fun": 848,
+  " function": 1589,
+  " functions": 3658,
+  " funeral": 3666,
+  " funny": 1023,
+  " further": 1042,
+  " future": 941,
+  " gain": 3115,
+  " gained": 3390,
+  " game": 485,
+  " games": 1021,
+  " garden": 1666,
+  " gas": 1721,
+  " gate": 2783,
+  " gathered": 3759,
+  " gave": 453,
+  " gay": 2284,
+  " general": 861,
+  " generally": 1935,
+  " generation": 2943,
+  " gentle": 4001,
+  " gentleman": 2633,
+  " gentlemen": 2147,
+  " genus": 3641,
+  " get": 75,
+  " gets": 771,
+  " getting": 396,
+  " ghost": 3951,
+  " giant": 2539,
+  " gift": 2351,
+  " girl": 373,
+  " girlfriend": 3036,
+  " girls": 858,
+  " give": 210,
+  " given": 544,
+  " gives": 1240,
+  " giving": 1066,
+  " glad": 1138,
+  " glass": 1764,
+  " glasses": 3754,
+  " global": 3858,
+  " go": 77,
+  " goal": 1914,
+  " goals": 2693,
+  " god": 1598,
+  " gods": 3334,
+  " goes": 514,
+  " going": 103,
+  " gold": 1079,
+  " golden": 2599,
+  " gone": 471,
+  " gonna": 162,
+  " good": 128,
+  " goodness": 3192,
+  " goods": 3227,
+  " got": 94,
+  " gotta": 756,
+  " gotten": 3279,
+  " government": 628,
+  " governor": 3542,
+  " grab": 3907,
+  " grade": 3805,
+  " graduated": 3626,
+  " grand": 2774,
+  " grandfather": 4022,
+  " granted": 3829,
+  " grass": 2553,
+  " grateful": 3953,
+  " grave": 3055,
+  " gray": 3947,
+  " great": 233,
+  " greater": 2004,
+  " greatest": 1973,
+  " greatly": 4036,
+  " green": 1049,
+  " grew": 1245,
+  " ground": 896,
+  " group": 564,
+  " groups": 1351,
+  " grow": 1427,
+  " growing": 1839,
+  " grown": 2365,
+  " growth": 2717,
+  " guard": 2756,
+  " guess": 652,
+  " guest": 2885,
+  " guests": 3649,
+  " guilty": 2727,
+  " guitar": 3379,
+  " gun": 1158,
+  " guns": 2395,
+  " guy": 532,
+  " guys": 538,
+  " ha": 2071,
+  " had": 54,
+  " hadn't": 2356,
+  " hair": 825,
+  " half": 516,
+  " hall": 2119,
+  " hand": 378,
+  " handle": 2043,
+  " hands": 574,
+  " handsome": 3161,
+  " hang": 1953,
+  " hanging": 2753,
+  " happen": 696,
+  " happened": 386,
+  " happening": 1392,
+  " happens": 975,
+  " happiness": 2953,
+  " happy": 609,
+  " hard": 463,
+  " harder": 3466,
+  " hardly": 2072,
+  " harm": 3080,
+  " has": 81,
+  " hasn't": 1782,
+  " hat": 1626,
+  " hate": 1250,
+  " have": 28,
+  " haven't": 582,
+  " having": 433,
+  " he": 29,
+  " he'd": 1755,
+  " he'll": 1860,
+  " he's": 267,
+  " head": 321,
+  " headed": 3789,
+  " headquarters": 3653,
+  " heads": 2088,
+  " health": 1327,
+  " healthy": 3640,
+  " hear": 408,
+  " heard": 414,
+  " hearing": 2527,
+  " heart": 469,
+  " hearts": 3521,
+  " heat": 2127,
+  " heaven": 2754,
+  " heavily": 3875,
+  " heavy": 1374,
+  " height": 3190,
+  " held": 659,
+  " hell": 768,
+  " hello": 2859,
+  " help": 247,
+  " helped": 1354,
+  " helping": 2423,
+  " helps": 2920,
+  " her": 53,
+  " here": 76,
+  " here's": 2019,
+  " hero": 2701,
+  " herself": 956,
+  " hey": 1634,
+  " hi": 3967,
+  " hidden": 2823,
+  " hide": 1999,
+  " hiding": 2833,
+  " high": 495,
+  " higher": 1442,
+  " highest": 2001,
+  " highly": 2666,
+  " hill": 2273,
+  " hills": 3590,
+  " him": 63,
+  " himself": 424,
+  " hired": 3332,
+  " his": 38,
+  " historical": 3222,
+  " history": 758,
+  " hit": 754,
+  " hm": 4016,
+  " hockey": 3621,
+  " hold": 670,
+  " holding": 1672,
+  " holds": 3121,
+  " hole": 1940,
+  " holiday": 3316,
+  " home": 218,
+  " homes": 3219,
+  " honest": 2189,
+  " honey": 1197,
+  " honor": 2240,
+  " honour": 3603,
+  " hope": 618,
+  " hoped": 4066,
+  " hoping": 3044,
+  " horrible": 3228,
+  " horse": 1139,
+  " horses": 2079,
+  " hospital": 1166,
+  " host": 2811,
+  " hot": 964,
+  " hotel": 2027,
+  " hour": 860,
+  " hours": 737,
+  " house": 278,
+  " houses": 1820,
+  " how": 142,
+  " however": 934,
+  " huge": 1522,
+  " huh": 851,
+  " human": 718,
+  " humans": 2433,
+  " hundred": 646,
+  " hundreds": 3059,
+  " hung": 3167,
+  " hungry": 1922,
+  " hunting": 3091,
+  " hurry": 2125,
+  " hurt": 885,
+  " hurts": 4018,
+  " husband": 838,
+  " i": 263,
+  " i'll": 2310,
+  " i'm": 1568,
+  " ice": 1312,
+  " idea": 448,
+  " ideas": 1576,
+  " identified": 3945,
+  " identity": 3525,
+  " idiot": 3095,
+  " if": 78,
+  " ill": 2462,
+  " illegal": 4067,
+  " image": 2173,
+  " images": 3985,
+  " imagine": 1652,
+  " immediately": 1586,
+  " impact": 3433,
+  " importance": 3168,
+  " important": 452,
+  " impossible": 1894,
+  " improve": 3802,
+  " in": 13,
+  " incident": 3845,
+  " include": 943,
+  " included": 1275,
+  " includes": 1791,
+  " including": 694,
+  " income": 3040,
+  " increase": 2044,
+  " increased": 2609,
+  " increasing": 4030,
+  " incredible": 3995,
+  " indeed": 1317,
+  " independence": 3786,
+  " independent": 2191,
+  " individual": 1775,
+  " individuals": 3124,
+  " industrial": 3780,
+  " industry": 1967,
+  " influence": 2156,
+  " information": 856,
+  " inhabitants": 3371,
+  " initially": 4061,
+  " injured": 3771,
+  " injury": 3702,
+  " innocent": 2950,
+  " inside": 677,
+  " inspired": 3944,
+  " instance": 2738,
+  " instant": 3785,
+  " instead": 940,
+  " insurance": 3287,
+  " intelligence": 3804,
+  " intended": 3157,
+  " interest": 1111,
+  " interested": 1389,
+  " interesting": 1095,
+  " interests": 3620,
+  " international": 1540,
+  " interview": 2888,
+  " into": 125,
+  " introduce": 3586,
+  " introduced": 2166,
+  " invented": 3970,
+  " investigation": 3302,
+  " invited": 2663,
+  " involved": 1188,
+  " iron": 2522,
+  " is": 16,
+  " island": 1235,
+  " islands": 3508,
+  " isn't": 407,
+  " issue": 1689,
+  " issued": 3775,
+  " issues": 1995,
+  " it": 18,
+  " it ": 3221,
+  " it'll": 1886,
+  " it's": 97,
+  " items": 3712,
+  " its": 173,
+  " itself": 952,
+  " jail": 2741,
+  " job": 489,
+  " jobs": 2466,
+  " join": 1557,
+  " joined": 1214,
+  " joint": 3863,
+  " joke": 2448,
+  " journalist": 3930,
+  " journey": 2363,
+  " joy": 2710,
+  " judge": 2274,
+  " juice": 3026,
+  " jump": 2205,
+  " jumped": 3687,
+  " just": 73,
+  " justice": 2900,
+  " keep": 318,
+  " keeping": 2179,
+  " keeps": 2619,
+  " kept": 897,
+  " key": 1290,
+  " keys": 3212,
+  " kick": 3003,
+  " kid": 1105,
+  " kidding": 2608,
+  " kids": 875,
+  " kill": 561,
+  " killed": 594,
+  " killer": 3075,
+  " killing": 2100,
+  " kind": 275,
+  " kinda": 4070,
+  " kinds": 1892,
+  " king": 931,
+  " kingdom": 3025,
+  " kiss": 1816,
+  " kitchen": 1875,
+  " knees": 3788,
+  " knew": 430,
+  " knife": 2716,
+  " knock": 3331,
+  " knocked": 3784,
+  " know": 46,
+  " knowing": 2358,
+  " knowledge": 1738,
+  " known": 271,
+  " knows": 791,
+  " l": 2832,
+  " la": 1731,
+  " label": 3474,
+  " labor": 3801,
+  " lack": 2768,
+  " lad": 3595,
+  " ladies": 2159,
+  " lady": 967,
+  " laid": 1876,
+  " lake": 2662,
+  " land": 672,
+  " lands": 3707,
+  " language": 909,
+  " languages": 2202,
+  " large": 600,
+  " larger": 1934,
+  " largest": 1222,
+  " last": 216,
+  " late": 596,
+  " later": 323,
+  " latter": 2919,
+  " laugh": 1726,
+  " laughed": 2246,
+  " laughing": 2300,
+  " launched": 3182,
+  " law": 835,
+  " laws": 2299,
+  " lawyer": 2206,
+  " lay": 1135,
+  " lead": 1048,
+  " leader": 1486,
+  " leaders": 2980,
+  " leading": 1793,
+  " leads": 3506,
+  " league": 2315,
+  " learn": 1019,
+  " learned": 1505,
+  " learning": 1982,
+  " least": 577,
+  " leave": 371,
+  " leaves": 1528,
+  " leaving": 1046,
+  " led": 991,
+  " left": 264,
+  " leg": 2158,
+  " legal": 2238,
+  " legs": 1541,
+  " length": 1580,
+  " less": 645,
+  " lesson": 3549,
+  " let": 262,
+  " let's": 478,
+  " letter": 985,
+  " letters": 1724,
+  " letting": 3850,
+  " level": 953,
+  " levels": 2802,
+  " library": 2722,
+  " lie": 1378,
+  " lies": 2124,
+  " life": 197,
+  " lift": 3315,
+  " lifted": 3929,
+  " light": 569,
+  " lights": 2497,
+  " like": 59,
+  " liked": 1535,
+  " likely": 1593,
+  " likes": 1952,
+  " limit": 3766,
+  " limited": 2874,
+  " line": 562,
+  " lines": 1506,
+  " link": 3781,
+  " lips": 2549,
+  " list": 960,
+  " listed": 3139,
+  " listen": 881,
+  " listened": 3618,
+  " listening": 2218,
+  " literally": 3973,
+  " literature": 3368,
+  " little": 132,
+  " live": 382,
+  " lived": 691,
+  " lives": 804,
+  " living": 665,
+  " load": 3535,
+  " local": 841,
+  " located": 1122,
+  " location": 2360,
+  " lock": 3023,
+  " locked": 2952,
+  " lonely": 3326,
+  " long": 206,
+  " longer": 842,
+  " look": 167,
+  " looked": 517,
+  " looking": 380,
+  " looks": 656,
+  " loose": 3372,
+  " lord": 2935,
+  " lose": 1092,
+  " losing": 2372,
+  " loss": 2074,
+  " lost": 496,
+  " lot": 255,
+  " lots": 1704,
+  " loud": 2778,
+  " love": 231,
+  " loved": 1251,
+  " lovely": 1450,
+  " loves": 2381,
+  " low": 1112,
+  " lower": 1573,
+  " luck": 1546,
+  " lucky": 1899,
+  " lunch": 1929,
+  " lying": 1612,
+  " m": 2571,
+  " ma'am": 3143,
+  " machine": 1534,
+  " machines": 3921,
+  " mad": 1679,
+  " made": 163,
+  " magazine": 2430,
+  " magic": 2311,
+  " mail": 3879,
+  " main": 747,
+  " mainly": 2257,
+  " major": 944,
+  " majority": 2577,
+  " make": 145,
+  " makes": 621,
+  " making": 509,
+  " male": 2149,
+  " man": 157,
+  " man's": 2668,
+  " manage": 2948,
+  " managed": 2259,
+  " management": 2582,
+  " manager": 1951,
+  " manner": 2353,
+  " many": 186,
+  " map": 2968,
+  " mark": 2412,
+  " marked": 3360,
+  " market": 1388,
+  " marks": 3727,
+  " marriage": 1425,
+  " married": 587,
+  " marry": 1547,
+  " mass": 2111,
+  " massive": 3899,
+  " master": 1655,
+  " match": 1181,
+  " matches": 2426,
+  " mate": 2616,
+  " material": 1819,
+  " materials": 3314,
+  " matter": 479,
+  " matters": 2471,
+  " may": 234,
+  " maybe": 591,
+  " mayor": 3741,
+  " me": 34,
+  " meal": 3234,
+  " mean": 176,
+  " meaning": 1572,
+  " means": 460,
+  " meant": 1115,
+  " measure": 2797,
+  " meat": 2327,
+  " medal": 3076,
+  " media": 2130,
+  " medical": 1845,
+  " medicine": 2607,
+  " meet": 640,
+  " meeting": 1030,
+  " member": 684,
+  " members": 820,
+  " memory": 1742,
+  " men": 342,
+  " men's": 3792,
+  " mental": 2855,
+  " mention": 2383,
+  " mentioned": 1887,
+  " merely": 3289,
+  " mess": 2392,
+  " message": 1487,
+  " met": 769,
+  " metal": 2347,
+  " meters": 3592,
+  " method": 2477,
+  " methods": 3343,
+  " middle": 1064,
+  " might": 252,
+  " mile": 3683,
+  " miles": 1167,
+  " military": 1203,
+  " milk": 1639,
+  " million": 695,
+  " millions": 3652,
+  " mind": 403,
+  " minds": 3948,
+  " mine": 847,
+  " minister": 2671,
+  " minor": 3220,
+  " minus": 1963,
+  " minute": 678,
+  " minutes": 657,
+  " mirror": 3823,
+  " miss": 1411,
+  " missed": 2057,
+  " missing": 1735,
+  " mission": 2316,
+  " mistake": 1772,
+  " mixed": 2702,
+  " mm": 1964,
+  " model": 1432,
+  " models": 3822,
+  " modern": 1326,
+  " mom": 1285,
+  " moment": 548,
+  " moments": 3873,
+  " mommy": 1498,
+  " money": 286,
+  " monster": 3421,
+  " month": 1142,
+  " months": 726,
+  " mood": 3915,
+  " moon": 2095,
+  " moral": 3725,
+  " more": 102,
+  " morning": 431,
+  " most": 205,
+  " mostly": 1609,
+  " mother": 387,
+  " mother's": 2990,
+  " motion": 2862,
+  " mountain": 1848,
+  " mountains": 2652,
+  " mouse": 3836,
+  " mouth": 1044,
+  " move": 543,
+  " moved": 753,
+  " movement": 1510,
+  " moves": 3179,
+  " movie": 501,
+  " movies": 1494,
+  " moving": 1127,
+  " much": 156,
+  " multiple": 2758,
+  " mum": 2921,
+  " municipality": 1556,
+  " murder": 1561,
+  " murdered": 3599,
+  " museum": 3146,
+  " music": 466,
+  " musical": 2500,
+  " musician": 4041,
+  " must": 214,
+  " my": 49,
+  " myself": 584,
+  " mystery": 4056,
+  " n": 2902,
+  " n't": 443,
+  " name": 222,
+  " named": 664,
+  " names": 1360,
+  " narrow": 3478,
+  " nation": 2632,
+  " national": 749,
+  " native": 2535,
+  " natural": 1215,
+  " naturally": 4044,
+  " nature": 1295,
+  " near": 620,
+  " nearby": 3297,
+  " nearly": 1310,
+  " necessarily": 3867,
+  " necessary": 1617,
+  " neck": 1890,
+  " need": 168,
+  " needed": 1150,
+  " needs": 880,
+  " negative": 2099,
+  " neighborhood": 3731,
+  " neither": 2051,
+  " nervous": 2744,
+  " net": 4055,
+  " network": 2258,
+  " never": 169,
+  " new": 203,
+  " news": 900,
+  " newspaper": 2563,
+  " next": 307,
+  " nice": 418,
+  " night": 253,
+  " nights": 3545,
+  " nine": 1024,
+  " nineteen": 2613,
+  " ninety": 2786,
+  " no": 83,
+  " nobody": 1478,
+  " noise": 2139,
+  " nominated": 3189,
+  " none": 1563,
+  " nor": 1428,
+  " normal": 1461,
+  " normally": 2925,
+  " north": 911,
+  " northern": 2155,
+  " northwest": 3849,
+  " nose": 1560,
+  " not": 35,
+  " note": 1869,
+  " noted": 3102,
+  " notes": 2700,
+  " nothing": 315,
+  " notice": 1733,
+  " noticed": 2338,
+  " novel": 2197,
+  " now": 100,
+  " nuclear": 3200,
+  " number": 319,
+  " numbers": 1257,
+  " numerous": 3363,
+  " nurse": 3448,
+  " nuts": 3830,
+  " o": 3047,
+  " o'clock": 2370,
+  " object": 2020,
+  " objects": 2994,
+  " observed": 3456,
+  " obvious": 3166,
+  " obviously": 1850,
+  " occasion": 3679,
+  " occupied": 4009,
+  " occur": 3810,
+  " occurred": 2880,
+  " ocean": 3019,
+  " odd": 3150,
+  " of": 9,
+  " off": 165,
+  " offer": 1532,
+  " offered": 2033,
+  " office": 783,
+  " officer": 1700,
+  " officers": 2357,
+  " official": 1640,
+  " officially": 3135,
+  " often": 511,
+  " oh": 616,
+  " oil": 1767,
+  " ok": 3249,
+  " okay": 296,
+  " old": 217,
+  " older": 1606,
+  " oldest": 2822,
+  " on": 23,
+  " once": 362,
+  " one": 52,
+  " one's": 1944,
+  " ones": 852,
+  " online": 2974,
+  " only": 139,
+  " onto": 2141,
+  " open": 508,
+  " opened": 1034,
+  " opening": 1896,
+  " opera": 3485,
+  " operated": 4050,
+  " operating": 3540,
+  " operation": 2407,
+  " operations": 2973,
+  " opinion": 2233,
+  " opportunity": 1814,
+  " opposed": 3923,
+  " opposite": 2394,
+  " or": 69,
+  " orange": 3106,
+  " order": 586,
+  " ordered": 2058,
+  " orders": 2021,
+  " ordinary": 3073,
+  " organization": 2190,
+  " organizations": 4013,
+  " organized": 3384,
+  " origin": 3636,
+  " original": 1072,
+  " originally": 2039,
+  " other": 127,
+  " others": 666,
+  " otherwise": 2628,
+  " ought": 1642,
+  " our": 140,
+  " ours": 3385,
+  " ourselves": 2062,
+  " out": 66,
+  " outside": 721,
+  " over": 129,
+  " overall": 3235,
+  " owe": 3231,
+  " own": 245,
+  " owned": 2326,
+  " owner": 2851,
+  " p": 2290,
+  " pack": 3502,
+  " page": 1625,
+  " pages": 3532,
+  " paid": 1195,
+  " pain": 1451,
+  " paint": 3058,
+  " painted": 3583,
+  " painting": 2598,
+  " pair": 2059,
+  " palace": 2245,
+  " pale": 3594,
+  " pants": 3606,
+  " paper": 915,
+  " papers": 2364,
+  " parents": 965,
+  " parish": 3601,
+  " park": 1664,
+  " part": 246,
+  " particular": 1110,
+  " particularly": 1889,
+  " parties": 2875,
+  " partner": 2406,
+  " parts": 1067,
+  " party": 681,
+  " pass": 1173,
+  " passed": 1002,
+  " passing": 2649,
+  " passion": 4060,
+  " past": 763,
+  " path": 2009,
+  " patient": 2182,
+  " patients": 3184,
+  " pattern": 3256,
+  " pay": 638,
+  " paying": 2661,
+  " peace": 1524,
+  " pen": 3891,
+  " people": 113,
+  " people's": 3514,
+  " per": 1156,
+  " percent": 1677,
+  " perfect": 1381,
+  " perfectly": 2555,
+  " perform": 3209,
+  " performance": 1939,
+  " performed": 1976,
+  " perhaps": 932,
+  " period": 1007,
+  " permission": 3477,
+  " person": 444,
+  " personal": 1363,
+  " personality": 4037,
+  " personally": 3906,
+  " persons": 3744,
+  " philosophy": 3614,
+  " phone": 824,
+  " photo": 3751,
+  " physical": 2132,
+  " piano": 3012,
+  " pick": 982,
+  " picked": 1884,
+  " picking": 4049,
+  " picture": 954,
+  " pictures": 1794,
+  " piece": 951,
+  " pieces": 1611,
+  " pig": 3241,
+  " pilot": 3928,
+  " pink": 3078,
+  " pity": 3613,
+  " place": 236,
+  " placed": 1743,
+  " places": 1089,
+  " plain": 2886,
+  " plan": 913,
+  " plane": 1965,
+  " planet": 2144,
+  " planned": 2601,
+  " planning": 2276,
+  " plans": 1906,
+  " plant": 1807,
+  " plants": 1993,
+  " plastic": 3911,
+  " plate": 2914,
+  " platform": 3630,
+  " play": 358,
+  " played": 420,
+  " player": 814,
+  " players": 1777,
+  " playing": 711,
+  " plays": 1196,
+  " pleasant": 3086,
+  " please": 392,
+  " pleased": 2641,
+  " pleasure": 1947,
+  " plenty": 2683,
+  " plot": 3952,
+  " plus": 1509,
+  " pocket": 2580,
+  " poem": 4008,
+  " poet": 3501,
+  " poetry": 3903,
+  " point": 415,
+  " pointed": 3548,
+  " points": 1258,
+  " police": 766,
+  " policy": 1827,
+  " political": 1057,
+  " politician": 1255,
+  " politics": 2918,
+  " pool": 3009,
+  " poor": 774,
+  " pop": 2570,
+  " popular": 1091,
+  " population": 788,
+  " port": 3677,
+  " position": 922,
+  " positions": 4004,
+  " positive": 1885,
+  " possibility": 3808,
+  " possible": 728,
+  " possibly": 2266,
+  " post": 2212,
+  " pot": 3977,
+  " potential": 2687,
+  " pound": 2503,
+  " pounds": 1406,
+  " power": 578,
+  " powerful": 1996,
+  " powers": 2747,
+  " practical": 3865,
+  " practice": 1660,
+  " pray": 3356,
+  " precious": 4062,
+  " prefer": 3480,
+  " pregnant": 3346,
+  " prepare": 3853,
+  " prepared": 2217,
+  " presence": 2494,
+  " present": 874,
+  " presented": 2848,
+  " president": 1241,
+  " presidential": 3892,
+  " press": 1932,
+  " pressure": 1762,
+  " pretend": 3436,
+  " pretty": 482,
+  " prevent": 2906,
+  " previous": 2108,
+  " previously": 3024,
+  " price": 1683,
+  " pride": 3374,
+  " priest": 3278,
+  " primary": 2620,
+  " prime": 3340,
+  " prince": 1942,
+  " princess": 3066,
+  " principle": 3965,
+  " print": 3958,
+  " prison": 1629,
+  " prisoners": 3732,
+  " private": 1328,
+  " prize": 3492,
+  " probably": 541,
+  " problem": 521,
+  " problems": 945,
+  " process": 1137,
+  " produce": 2168,
+  " produced": 1216,
+  " producer": 2694,
+  " product": 2251,
+  " production": 1555,
+  " products": 2712,
+  " professional": 1239,
+  " professor": 2704,
+  " program": 1263,
+  " programme": 3663,
+  " programs": 2624,
+  " progress": 3233,
+  " project": 1452,
+  " projects": 3750,
+  " promise": 1349,
+  " promised": 2118,
+  " proof": 3048,
+  " proper": 2450,
+  " properly": 3088,
+  " properties": 4078,
+  " property": 1741,
+  " proposed": 2937,
+  " protect": 1676,
+  " protection": 3028,
+  " proud": 1950,
+  " prove": 2133,
+  " proved": 3271,
+  " provide": 1874,
+  " provided": 2288,
+  " provides": 3697,
+  " province": 1883,
+  " public": 715,
+  " published": 1125,
+  " pull": 1467,
+  " pulled": 2171,
+  " punishment": 4063,
+  " pure": 3089,
+  " purpose": 1769,
+  " push": 1956,
+  " pushed": 3118,
+  " put": 174,
+  " puts": 3769,
+  " putting": 1479,
+  " quality": 2229,
+  " quarter": 2757,
+  " queen": 2729,
+  " question": 641,
+  " questions": 1076,
+  " quick": 1558,
+  " quickly": 1096,
+  " quiet": 1403,
+  " quietly": 3582,
+  " quit": 3071,
+  " quite": 381,
+  " race": 1180,
+  " racing": 3696,
+  " radio": 1488,
+  " railway": 2679,
+  " rain": 1924,
+  " raise": 2411,
+  " raised": 1600,
+  " ran": 923,
+  " range": 1698,
+  " rank": 3861,
+  " rare": 3011,
+  " rate": 1720,
+  " rather": 648,
+  " reach": 1379,
+  " reached": 1071,
+  " reaction": 3335,
+  " read": 555,
+  " reading": 1670,
+  " ready": 590,
+  " real": 451,
+  " reality": 2397,
+  " realize": 2050,
+  " realized": 2771,
+  " really": 152,
+  " reason": 692,
+  " reasons": 2047,
+  " receive": 2498,
+  " received": 882,
+  " recent": 2530,
+  " recently": 2227,
+  " recognize": 3068,
+  " recognized": 3006,
+  " record": 950,
+  " recorded": 1717,
+  " recording": 2794,
+  " records": 2262,
+  " red": 780,
+  " reduced": 3824,
+  " refer": 2467,
+  " reference": 3625,
+  " referred": 2958,
+  " refused": 2865,
+  " regard": 4054,
+  " regarded": 4076,
+  " region": 816,
+  " regional": 4015,
+  " regions": 3588,
+  " regular": 1749,
+  " related": 1955,
+  " relations": 3645,
+  " relationship": 1519,
+  " release": 1678,
+  " released": 601,
+  " relief": 3726,
+  " religion": 2410,
+  " religious": 2188,
+  " remain": 2089,
+  " remained": 1857,
+  " remaining": 3277,
+  " remains": 2325,
+  " remember": 439,
+  " remembered": 2795,
+  " remove": 3749,
+  " removed": 2592,
+  " rent": 3979,
+  " repeated": 3844,
+  " replace": 3857,
+  " replaced": 2087,
+  " replied": 1691,
+  " reply": 4021,
+  " report": 1207,
+  " reported": 2135,
+  " reports": 3119,
+  " represent": 3178,
+  " represented": 2667,
+  " request": 3309,
+  " require": 4003,
+  " required": 2334,
+  " rescue": 4020,
+  " research": 1338,
+  " resources": 2767,
+  " respect": 1622,
+  " response": 2526,
+  " responsibility": 2846,
+  " responsible": 2076,
+  " rest": 637,
+  " restaurant": 3117,
+  " result": 1176,
+  " results": 1872,
+  " retired": 1994,
+  " return": 907,
+  " returned": 1037,
+  " returning": 3840,
+  " revealed": 3568,
+  " review": 3439,
+  " reviews": 3753,
+  " rich": 1473,
+  " rid": 2232,
+  " ride": 1518,
+  " ridiculous": 3310,
+  " riding": 3585,
+  " right": 86,
+  " rights": 1376,
+  " ring": 1318,
+  " rings": 3431,
+  " rise": 2501,
+  " rising": 3791,
+  " risk": 1986,
+  " river": 1035,
+  " rivers": 4075,
+  " road": 806,
+  " roads": 3517,
+  " rock": 1104,
+  " rocks": 2873,
+  " role": 1055,
+  " roles": 2634,
+  " roll": 2568,
+  " romantic": 3395,
+  " roof": 2483,
+  " room": 400,
+  " rooms": 3045,
+  " root": 3420,
+  " rose": 1900,
+  " rough": 3015,
+  " round": 533,
+  " route": 2699,
+  " row": 2725,
+  " royal": 3128,
+  " rule": 1554,
+  " ruled": 3544,
+  " rules": 1682,
+  " run": 497,
+  " running": 867,
+  " runs": 1865,
+  " rush": 3839,
+  " s": 2565,
+  " sad": 1756,
+  " safe": 1143,
+  " safety": 2499,
+  " said": 96,
+  " sake": 2077,
+  " sale": 3441,
+  " sales": 3207,
+  " salt": 3693,
+  " same": 224,
+  " sand": 3240,
+  " sang": 3827,
+  " sat": 1050,
+  " satisfied": 4043,
+  " save": 1014,
+  " saved": 1908,
+  " saw": 316,
+  " say": 135,
+  " saying": 447,
+  " says": 374,
+  " scale": 2782,
+  " scared": 1564,
+  " scene": 1653,
+  " scheme": 3943,
+  " school": 348,
+  " schools": 1480,
+  " science": 1475,
+  " scientific": 2891,
+  " scientists": 3444,
+  " score": 2597,
+  " scored": 2612,
+  " screen": 2759,
+  " sea": 937,
+  " search": 1693,
+  " season": 660,
+  " seasons": 2824,
+  " seat": 1031,
+  " seats": 3383,
+  " second": 329,
+  " seconds": 2162,
+  " secret": 1237,
+  " secretary": 4048,
+  " section": 2073,
+  " secure": 3904,
+  " security": 1787,
+  " see": 91,
+  " seeing": 1229,
+  " seek": 3108,
+  " seem": 994,
+  " seemed": 830,
+  " seems": 812,
+  " seen": 337,
+  " sees": 2835,
+  " selected": 2509,
+  " self": 3783,
+  " sell": 1397,
+  " selling": 2695,
+  " send": 981,
+  " sending": 3304,
+  " senior": 3137,
+  " sense": 846,
+  " sent": 700,
+  " sentence": 2978,
+  " separate": 2075,
+  " separated": 3889,
+  " series": 629,
+  " serious": 1201,
+  " seriously": 2752,
+  " servant": 3898,
+  " serve": 2036,
+  " served": 961,
+  " service": 873,
+  " services": 1631,
+  " serving": 3054,
+  " set": 390,
+  " sets": 3313,
+  " setting": 3007,
+  " settle": 3887,
+  " settled": 2588,
+  " settlement": 3293,
+  " seven": 784,
+  " several": 626,
+  " severe": 4065,
+  " sex": 1457,
+  " sexual": 2917,
+  " shadow": 3348,
+  " shake": 3651,
+  " shall": 519,
+  " shame": 3213,
+  " shape": 2016,
+  " share": 1384,
+  " shared": 3341,
+  " sharp": 2949,
+  " she": 74,
+  " she'd": 3357,
+  " she'll": 3197,
+  " she's": 505,
+  " sheep": 3094,
+  " ship": 1033,
+  " ships": 2469,
+  " shirt": 2970,
+  " shit": 1026,
+  " shock": 3819,
+  " shoes": 1835,
+  " shook": 3366,
+  " shoot": 1650,
+  " shooting": 2737,
+  " shop": 1739,
+  " shopping": 3672,
+  " shore": 3004,
+  " short": 776,
+  " shortly": 3900,
+  " shot": 912,
+  " should": 171,
+  " shoulder": 2946,
+  " shoulders": 3689,
+  " shouldn't": 1400,
+  " show": 324,
+  " showed": 1383,
+  " shower": 4058,
+  " showing": 2292,
+  " shown": 1685,
+  " shows": 1175,
+  " shut": 1343,
+  " sick": 1179,
+  " side": 399,
+  " sides": 2018,
+  " sight": 1530,
+  " sign": 1200,
+  " signal": 2659,
+  " signed": 1599,
+  " significant": 2548,
+  " signs": 3114,
+  " silence": 2543,
+  " silent": 2821,
+  " silly": 2140,
+  " silver": 2126,
+  " similar": 1178,
+  " simple": 1129,
+  " simply": 1302,
+  " since": 375,
+  " sing": 1566,
+  " singer": 1753,
+  " singing": 2037,
+  " single": 725,
+  " sir": 442,
+  " sister": 892,
+  " sisters": 3082,
+  " sit": 739,
+  " site": 1468,
+  " sites": 3674,
+  " sitting": 1086,
+  " situation": 1162,
+  " six": 524,
+  " sixth": 3657,
+  " sixty": 2829,
+  " size": 1266,
+  " skills": 3404,
+  " skin": 1831,
+  " sky": 1734,
+  " sleep": 759,
+  " sleeping": 2302,
+  " slept": 3185,
+  " slide": 3729,
+  " slightly": 2733,
+  " slow": 2148,
+  " slowly": 1881,
+  " small": 436,
+  " smaller": 1852,
+  " smart": 2349,
+  " smell": 2239,
+  " smile": 1931,
+  " smiled": 3646,
+  " smiling": 3890,
+  " smoke": 2428,
+  " snow": 2098,
+  " so": 65,
+  " social": 1116,
+  " society": 1496,
+  " soft": 2063,
+  " software": 3170,
+  " soil": 3419,
+  " sold": 1211,
+  " soldier": 2894,
+  " soldiers": 1730,
+  " solid": 3409,
+  " solution": 2587,
+  " solve": 3205,
+  " some": 105,
+  " somebody": 930,
+  " somehow": 2706,
+  " someone": 546,
+  " something": 160,
+  " sometimes": 722,
+  " somewhat": 3126,
+  " somewhere": 1247,
+  " son": 419,
+  " song": 633,
+  " songs": 1499,
+  " sons": 2749,
+  " soon": 487,
+  " sooner": 3821,
+  " sorry": 354,
+  " sort": 468,
+  " sorts": 3402,
+  " soul": 1495,
+  " sound": 797,
+  " sounds": 1352,
+  " soup": 4071,
+  " source": 2008,
+  " sources": 3633,
+  " south": 1148,
+  " southern": 2268,
+  " space": 1000,
+  " spare": 3541,
+  " speak": 777,
+  " speaking": 1680,
+  " special": 833,
+  " species": 933,
+  " specific": 2304,
+  " speech": 2112,
+  " speed": 1813,
+  " spell": 3519,
+  " spend": 1508,
+  " spending": 3565,
+  " spent": 1198,
+  " spirit": 1879,
+  " spite": 3411,
+  " split": 2547,
+  " spoke": 1438,
+  " spoken": 2322,
+  " sport": 3577,
+  " sports": 2405,
+  " spot": 1809,
+  " spread": 1926,
+  " spring": 1991,
+  " square": 1607,
+  " squared": 3629,
+  " staff": 1949,
+  " stage": 1117,
+  " stairs": 3559,
+  " stand": 864,
+  " standard": 2223,
+  " standing": 1252,
+  " stands": 2966,
+  " star": 1521,
+  " starring": 3761,
+  " stars": 1308,
+  " start": 445,
+  " started": 423,
+  " starting": 1439,
+  " starts": 1943,
+  " state": 446,
+  " stated": 2872,
+  " statement": 2455,
+  " states": 1602,
+  " station": 862,
+  " stations": 2884,
+  " statistics": 2092,
+  " status": 2642,
+  " stay": 510,
+  " stayed": 2314,
+  " staying": 2831,
+  " steal": 3069,
+  " steel": 3531,
+  " step": 1254,
+  " steps": 2297,
+  " stick": 1737,
+  " still": 196,
+  " stock": 3226,
+  " stole": 3202,
+  " stolen": 3643,
+  " stomach": 3673,
+  " stone": 1647,
+  " stones": 3695,
+  " stood": 1041,
+  " stop": 457,
+  " stopped": 1164,
+  " stops": 4002,
+  " store": 1483,
+  " stories": 1441,
+  " storm": 2174,
+  " story": 520,
+  " straight": 1063,
+  " strange": 1253,
+  " stranger": 3547,
+  " stream": 3855,
+  " street": 1228,
+  " streets": 2581,
+  " strength": 1798,
+  " strike": 2834,
+  " string": 2991,
+  " strong": 886,
+  " stronger": 3311,
+  " struck": 2436,
+  " structure": 1961,
+  " struggle": 3489,
+  " stuck": 1972,
+  " student": 1804,
+  " students": 1085,
+  " studied": 1823,
+  " studies": 2332,
+  " studio": 1888,
+  " study": 1106,
+  " studying": 3608,
+  " stuff": 685,
+  " stupid": 1377,
+  " style": 1548,
+  " subject": 1502,
+  " subjects": 3825,
+  " succeeded": 3661,
+  " success": 1531,
+  " successful": 1641,
+  " such": 225,
+  " sudden": 2289,
+  " suddenly": 1359,
+  " suffer": 3698,
+  " suffered": 2883,
+  " suffering": 3064,
+  " sugar": 2815,
+  " suggest": 2852,
+  " suggested": 2655,
+  " suicide": 2988,
+  " suit": 2409,
+  " sum": 3579,
+  " summer": 1274,
+  " sun": 1189,
+  " supper": 3902,
+  " supply": 2805,
+  " support": 929,
+  " supported": 2762,
+  " suppose": 1128,
+  " supposed": 895,
+  " sure": 276,
+  " surely": 3358,
+  " surface": 1713,
+  " surgery": 3414,
+  " surprise": 1826,
+  " surprised": 2172,
+  " surrounded": 3735,
+  " survive": 3081,
+  " suspect": 3426,
+  " swear": 2401,
+  " sweet": 1316,
+  " sweetie": 3354,
+  " swim": 3835,
+  " swimming": 3266,
+  " switch": 3659,
+  " sword": 2595,
+  " symbol": 3748,
+  " system": 567,
+  " systems": 1818,
+  " t": 3224,
+  " table": 868,
+  " tail": 2151,
+  " take": 146,
+  " taken": 624,
+  " takes": 939,
+  " taking": 636,
+  " talk": 294,
+  " talked": 1333,
+  " talking": 437,
+  " talks": 3604,
+  " tall": 2052,
+  " tape": 1998,
+  " target": 3090,
+  " task": 2963,
+  " taste": 2187,
+  " taught": 1796,
+  " tax": 2586,
+  " tea": 1445,
+  " teach": 1861,
+  " teacher": 1709,
+  " teachers": 3285,
+  " teaching": 2496,
+  " team": 410,
+  " teams": 1752,
+  " tears": 2529,
+  " technology": 2181,
+  " teeth": 2045,
+  " telephone": 2934,
+  " television": 887,
+  " tell": 181,
+  " telling": 987,
+  " tells": 1662,
+  " temperature": 2540,
+  " temple": 2979,
+  " ten": 668,
+  " tend": 3301,
+  " term": 1108,
+  " terms": 1159,
+  " terrible": 1549,
+  " territory": 3033,
+  " test": 1344,
+  " tests": 4014,
+  " text": 2514,
+  " th": 2869,
+  " than": 148,
+  " thank": 787,
+  " thanks": 1362,
+  " that": 17,
+  " that ": 3351,
+  " that's": 151,
+  " the": 7,
+  " the ": 2537,
+  " theatre": 3961,
+  " thee": 2391,
+  " their": 104,
+  " them": 80,
+  " theme": 3416,
+  " themselves": 906,
+  " then": 107,
+  " theory": 1619,
+  " there": 67,
+  " there's": 285,
+  " therefore": 1570,
+  " these": 154,
+  " they": 55,
+  " they'd": 2559,
+  " they'll": 1740,
+  " they're": 346,
+  " they've": 1311,
+  " thick": 2923,
+  " thin": 2933,
+  " thing": 189,
+  " things": 191,
+  " think": 84,
+  " thinking": 650,
+  " thinks": 1758,
+  " third": 761,
+  " thirty": 1582,
+  " this": 31,
+  " those": 192,
+  " thou": 1868,
+  " though": 402,
+  " thought": 208,
+  " thoughts": 2445,
+  " thousand": 946,
+  " thousands": 2441,
+  " threat": 3773,
+  " three": 194,
+  " threw": 2254,
+  " throat": 2879,
+  " through": 195,
+  " throughout": 1778,
+  " throw": 1307,
+  " throwing": 3982,
+  " thrown": 3406,
+  " thus": 1897,
+  " thy": 2789,
+  " ticket": 3526,
+  " tickets": 3964,
+  " tie": 3262,
+  " tied": 2996,
+  " tight": 2998,
+  " till": 778,
+  " time": 95,
+  " times": 383,
+  " tiny": 2444,
+  " tired": 1276,
+  " title": 1268,
+  " to": 8,
+  " today": 406,
+  " together": 343,
+  " told": 270,
+  " tomorrow": 859,
+  " tone": 2788,
+  " tongue": 2790,
+  " tonight": 904,
+  " too": 155,
+  " took": 303,
+  " tools": 3339,
+  " top": 588,
+  " total": 1107,
+  " totally": 2104,
+  " touch": 1082,
+  " touched": 2960,
+  " tough": 2686,
+  " tour": 2513,
+  " tournament": 2889,
+  " toward": 1945,
+  " towards": 1103,
+  " tower": 3017,
+  " town": 467,
+  " towns": 3148,
+  " toy": 3988,
+  " toys": 3370,
+  " track": 1526,
+  " tracks": 3245,
+  " trade": 1748,
+  " tradition": 3445,
+  " traditional": 2451,
+  " traffic": 2760,
+  " train": 1100,
+  " trained": 3570,
+  " training": 1422,
+  " trains": 3469,
+  " transfer": 3862,
+  " transferred": 4012,
+  " transport": 3488,
+  " travel": 2030,
+  " treat": 2611,
+  " treated": 2981,
+  " treatment": 2425,
+  " tree": 919,
+  " trees": 1315,
+  " trial": 2272,
+  " trick": 3275,
+  " tried": 744,
+  " tries": 3800,
+  " trip": 1958,
+  " troops": 2369,
+  " tropical": 3575,
+  " trouble": 884,
+  " truck": 2115,
+  " true": 552,
+  " truly": 3031,
+  " trust": 1265,
+  " truth": 902,
+  " try": 412,
+  " trying": 461,
+  " turn": 549,
+  " turned": 687,
+  " turning": 2136,
+  " turns": 2024,
+  " twelve": 1803,
+  " twenty": 963,
+  " twice": 1703,
+  " two": 120,
+  " type": 899,
+  " types": 1907,
+  " typical": 4077,
+  " ugly": 3398,
+  " uh": 314,
+  " um": 1053,
+  " unable": 3473,
+  " uncle": 2281,
+  " under": 311,
+  " understand": 425,
+  " understanding": 2735,
+  " understood": 2600,
+  " union": 2964,
+  " unique": 3261,
+  " unit": 2165,
+  " units": 2393,
+  " universe": 2816,
+  " university": 1983,
+  " unknown": 3057,
+  " unless": 1808,
+  " until": 298,
+  " unusual": 3422,
+  " up": 61,
+  " upon": 455,
+  " upper": 2801,
+  " upset": 2510,
+  " upstairs": 2707,
+  " urban": 4032,
+  " us": 136,
+  " use": 261,
+  " used": 198,
+  " useful": 2517,
+  " uses": 1849,
+  " using": 717,
+  " usual": 2286,
+  " usually": 767,
+  " v": 3779,
+  " valley": 3305,
+  " value": 1191,
+  " values": 3013,
+  " van": 3229,
+  " variety": 2910,
+  " various": 1194,
+  " vast": 3997,
+  " vehicle": 3498,
+  " version": 1375,
+  " very": 115,
+  " via": 3539,
+  " victim": 3401,
+  " victims": 3843,
+  " victory": 2643,
+  " video": 988,
+  " videos": 4033,
+  " view": 1184,
+  " views": 3787,
+  " village": 921,
+  " violence": 3035,
+  " violent": 4006,
+  " vision": 3169,
+  " visit": 1419,
+  " visited": 3518,
+  " voice": 731,
+  " voices": 3610,
+  " volume": 3543,
+  " vote": 1919,
+  " votes": 3922,
+  " wait": 560,
+  " waited": 2867,
+  " waiting": 920,
+  " wake": 2213,
+  " walk": 845,
+  " walked": 1636,
+  " walking": 1687,
+  " wall": 1279,
+  " walls": 2337,
+  " wanna": 518,
+  " want": 101,
+  " wanted": 351,
+  " wants": 671,
+  " war": 647,
+  " warm": 1707,
+  " warning": 3529,
+  " was": 20,
+  " wash": 2931,
+  " wasn't": 529,
+  " waste": 2215,
+  " watch": 790,
+  " watched": 2270,
+  " watching": 1358,
+  " water": 339,
+  " waters": 4010,
+  " wave": 3104,
+  " waves": 3566,
+  " way": 141,
+  " ways": 1147,
+  " we": 39,
+  " we'd": 1923,
+  " we'll": 566,
+  " we're": 258,
+  " we've": 612,
+  " weak": 2765,
+  " weapon": 2947,
+  " weapons": 2536,
+  " wear": 1350,
+  " wearing": 1915,
+  " weather": 2180,
+  " web": 3936,
+  " website": 3138,
+  " wedding": 1725,
+  " week": 589,
+  " weekend": 2809,
+  " weeks": 1009,
+  " weight": 1969,
+  " weird": 2091,
+  " welcome": 1610,
+  " well": 150,
+  " went": 209,
+  " were": 68,
+  " weren't": 1437,
+  " west": 1331,
+  " western": 2541,
+  " wet": 2492,
+  " what": 56,
+  " what's": 470,
+  " whatever": 928,
+  " wheel": 3423,
+  " when": 89,
+  " whenever": 3496,
+  " where": 144,
+  " where's": 3132,
+  " whether": 792,
+  " which": 87,
+  " while": 282,
+  " white": 608,
+  " who": 98,
+  " who's": 1151,
+  " whole": 405,
+  " whom": 1056,
+  " whose": 1102,
+  " why": 238,
+  " wide": 1585,
+  " widely": 3709,
+  " wife": 491,
+  " wild": 1712,
+  " will": 79,
+  " willing": 2435,
+  " win": 957,
+  " wind": 1353,
+  " window": 1157,
+  " windows": 2736,
+  " wine": 2211,
+  " wing": 3959,
+  " wings": 2895,
+  " winner": 3291,
+  " winning": 2373,
+  " winter": 2014,
+  " wire": 3917,
+  " wise": 3225,
+  " wish": 755,
+  " wished": 3263,
+  " with": 27,
+  " within": 658,
+  " without": 344,
+  " witness": 3369,
+  " wo": 3481,
+  " woman": 441,
+  " women": 622,
+  " women's": 2776,
+  " won": 488,
+  " won't": 345,
+  " wonder": 1193,
+  " wondered": 3942,
+  " wonderful": 1364,
+  " wondering": 2858,
+  " wood": 1768,
+  " woods": 3280,
+  " word": 502,
+  " words": 614,
+  " wore": 3321,
+  " work": 182,
+  " worked": 723,
+  " workers": 2635,
+  " working": 553,
+  " works": 793,
+  " world": 260,
+  " world's": 3145,
+  " worn": 4074,
+  " worried": 1846,
+  " worry": 828,
+  " worse": 1459,
+  " worst": 2312,
+  " worth": 1202,
+  " would": 82,
+  " would've": 3598,
+  " wouldn't": 563,
+  " wound": 3777,
+  " wounded": 3580,
+  " write": 839,
+  " writer": 1851,
+  " writing": 1192,
+  " written": 801,
+  " wrong": 458,
+  " wrote": 823,
+  " x": 1288,
+  " y": 2253,
+  " ya": 2280,
+  " yard": 3607,
+  " yards": 3553,
+  " ye": 2932,
+  " yeah": 335,
+  " year": 287,
+  " years": 183,
+  " yellow": 1658,
+  " yes": 449,
+  " yesterday": 1542,
+  " yet": 421,
+  " you": 14,
+  " you ": 2839,
+  " you'd": 898,
+  " you'll": 634,
+  " you're": 149,
+  " you've": 450,
+  " young": 434,
+  " younger": 2318,
+  " your": 48,
+  " yours": 1101,
+  " yourself": 531,
+  " youth": 2007,
+  " zero": 2214,
+  " |": 1340,
+  " £": 3655,
+  " –": 2142,
+  " —": 3596,
+  " ‘": 2465,
+  " “": 352,
+  " ♪": 570,
+  "!": 26,
+  "! ": 3100,
+  "!\"": 607,
+  "!”": 1301,
+  "\"": 51,
+  "\" ": 3628,
+  "\")": 1165,
+  "\"),": 3267,
+  "\").": 2781,
+  "\",": 265,
+  "\".": 226,
+  "\"?": 2208,
+  "#": 969,
+  "&": 736,
+  "'": 654,
+  "'Cause": 3312,
+  "'s": 2682,
+  "(": 201,
+  ")": 110,
+  ") ": 1465,
+  "),": 515,
+  ").": 385,
+  "):": 3000,
+  "*": 1715,
+  "+": 2944,
+  ",": 6,
+  ", ": 556,
+  ",\"": 273,
+  ",”": 576,
+  "-": 25,
+  "--": 2621,
+  "-I": 2022,
+  "-No": 2731,
+  "-What": 2775,
+  "-Yeah": 3701,
+  "-Yes": 3345,
+  "-You": 2838,
+  "-year-old": 3934,
+  ".": 5,
+  ". ": 301,
+  ".\"": 158,
+  ".\" ": 1084,
+  ".)": 3260,
+  ".,": 1418,
+  "..": 525,
+  "...": 43,
+  "...\"": 3156,
+  "....": 1830,
+  "...?": 2447,
+  ".]": 3507,
+  ".”": 523,
+  "/": 773,
+  "/&": 3933,
+  "1": 1916,
+  "2": 1927,
+  "3": 2995,
+  "4": 3746,
+  ":": 92,
+  ";": 93,
+  ";&": 1681,
+  ";/": 3807,
+  "<eod>": 3,
+  "<frg>": 4,
+  "<oov>": 1,
+  "<pad>": 0,
+  "<sod>": 2,
+  "=": 416,
+  ">": 3581,
+  "> ": 3405,
+  "?": 15,
+  "? ": 4028,
+  "?\"": 454,
+  "?\" ": 3760,
+  "?”": 1047,
+  "A": 184,
+  "AND": 2432,
+  "About": 2283,
+  "According": 2959,
+  "Actually": 2424,
+  "After": 750,
+  "Again": 3572,
+  "Ah": 786,
+  "All": 279,
+  "Alright": 1936,
+  "Also": 3794,
+  "Although": 2939,
+  "Am": 3524,
+  "An": 1306,
+  "And": 72,
+  "Another": 2403,
+  "Any": 2114,
+  "Anything": 3428,
+  "Anyway": 2446,
+  "Are": 389,
+  "Aren't": 3987,
+  "As": 494,
+  "At": 623,
+  "Aye": 2951,
+  "B": 2354,
+  "Back": 3574,
+  "Be": 1706,
+  "Because": 558,
+  "Before": 2081,
+  "Besides": 3684,
+  "Big": 3638,
+  "Both": 3272,
+  "Bring": 3692,
+  "But": 111,
+  "By": 1155,
+  "Bye": 2137,
+  "C": 1424,
+  "CHAPTER": 3288,
+  "Call": 3181,
+  "Can": 602,
+  "Can't": 1978,
+  "Captain": 3554,
+  "Club": 3046,
+  "Come": 251,
+  "Cos": 2427,
+  "Could": 1992,
+  "D": 1795,
+  "Dad": 2323,
+  "Damn": 2558,
+  "David": 3400,
+  "Dear": 4019,
+  "Did": 534,
+  "Didn't": 2518,
+  "Do": 274,
+  "Does": 1710,
+  "Don't": 268,
+  "Dr": 2386,
+  "During": 2241,
+  "E": 3085,
+  "Early": 2986,
+  "Eh": 3838,
+  "Er": 1083,
+  "Erm": 877,
+  "Even": 1146,
+  "Every": 1821,
+  "Everybody": 2845,
+  "Everyone": 2713,
+  "Everything": 2196,
+  "Exactly": 3511,
+  "Excuse": 1435,
+  "Father": 3247,
+  "Fine": 2331,
+  "First": 1601,
+  "Five": 3486,
+  "For": 503,
+  "Forget": 4040,
+  "Four": 3718,
+  "From": 1144,
+  "Fuck": 2507,
+  "Get": 526,
+  "Give": 1065,
+  "Go": 522,
+  "God": 1380,
+  "Good": 528,
+  "Got": 1911,
+  "Great": 1966,
+  "Ha": 3142,
+  "Has": 3350,
+  "Have": 733,
+  "He": 131,
+  "He'll": 2983,
+  "He's": 340,
+  "Hello": 740,
+  "Help": 2226,
+  "Her": 1583,
+  "Here": 644,
+  "Here's": 2344,
+  "Hey": 291,
+  "Hi": 1081,
+  "His": 871,
+  "History": 1790,
+  "Hmm": 2779,
+  "Hold": 1928,
+  "How": 227,
+  "How's": 2942,
+  "However": 2511,
+  "Huh": 2186,
+  "Hurry": 2806,
+  "I": 24,
+  "I'd": 968,
+  "I'll": 254,
+  "I'm": 118,
+  "I've": 372,
+  "If": 187,
+  "In": 164,
+  "Is": 347,
+  "Isn't": 2228,
+  "It": 133,
+  "It'll": 2871,
+  "It's": 114,
+  "Jack": 3743,
+  "James": 3956,
+  "Jesus": 3077,
+  "John": 1912,
+  "Just": 353,
+  "K": 4073,
+  "Keep": 1948,
+  "La": 3994,
+  "Last": 3216,
+  "Leave": 2440,
+  "Let": 617,
+  "Let's": 391,
+  "Life": 3678,
+  "Like": 917,
+  "Listen": 1174,
+  "Little": 3177,
+  "Look": 422,
+  "Looks": 2997,
+  "Love": 3571,
+  "M": 2603,
+  "MAN": 3815,
+  "Make": 2399,
+  "Man": 2230,
+  "Many": 2534,
+  "May": 1962,
+  "Maybe": 703,
+  "Me": 1925,
+  "Mhm": 1584,
+  "Michael": 3711,
+  "Miss": 2244,
+  "Mm": 500,
+  "Mom": 2487,
+  "More": 2743,
+  "Most": 2361,
+  "Mother": 3622,
+  "Move": 2853,
+  "Mr": 785,
+  "Mrs": 2184,
+  "Music": 3926,
+  "My": 281,
+  "Never": 1812,
+  "New": 2602,
+  "Next": 2473,
+  "Nice": 2277,
+  "No": 116,
+  "Nobody": 2618,
+  "Not": 426,
+  "Nothing": 1204,
+  "Now": 228,
+  "O": 2219,
+  "OH": 3667,
+  "OK": 1323,
+  "Of": 810,
+  "Oh": 117,
+  "Ok": 2476,
+  "Okay": 272,
+  "On": 512,
+  "Once": 2042,
+  "One": 539,
+  "Only": 1529,
+  "Ooh": 1984,
+  "Open": 3112,
+  "Or": 844,
+  "Other": 3274,
+  "Our": 1231,
+  "Out": 3561,
+  "Over": 2724,
+  "P": 3700,
+  "People": 1545,
+  "Perhaps": 2123,
+  "Peter": 3120,
+  "Please": 615,
+  "Probably": 3886,
+  "Put": 1858,
+  "R": 4064,
+  "Really": 1169,
+  "Remember": 2260,
+  "Right": 397,
+  "S": 682,
+  "Say": 2278,
+  "See": 918,
+  "Shall": 3443,
+  "She": 230,
+  "She's": 639,
+  "Shit": 2893,
+  "Should": 3330,
+  "Shut": 2533,
+  "Since": 1806,
+  "Sir": 1574,
+  "Sit": 3027,
+  "So": 109,
+  "Some": 997,
+  "Somebody": 3966,
+  "Someone": 2956,
+  "Something": 2225,
+  "Sometimes": 2384,
+  "Sorry": 936,
+  "Stay": 2519,
+  "Still": 2850,
+  "Stop": 1004,
+  "Sure": 1559,
+  "T": 3308,
+  "THE": 1990,
+  "Take": 855,
+  "Tell": 977,
+  "Thank": 428,
+  "Thanks": 938,
+  "That": 237,
+  "That's": 175,
+  "The": 47,
+  "Their": 3211,
+  "Then": 366,
+  "There": 266,
+  "There's": 459,
+  "These": 976,
+  "They": 211,
+  "They'll": 3452,
+  "They're": 690,
+  "They've": 3029,
+  "Think": 2639,
+  "This": 161,
+  "Those": 1783,
+  "Three": 2094,
+  "Time": 3877,
+  "To": 575,
+  "Today": 2521,
+  "Too": 3551,
+  "Try": 3244,
+  "Two": 1291,
+  "Uh": 757,
+  "Uh-huh": 817,
+  "Um": 1346,
+  "Until": 3984,
+  "Very": 1345,
+  "Wait": 914,
+  "Was": 1728,
+  "Watch": 3123,
+  "We": 137,
+  "We'll": 836,
+  "We're": 456,
+  "We've": 1098,
+  "Welcome": 2678,
+  "Well": 143,
+  "Were": 3864,
+  "What": 99,
+  "What's": 394,
+  "Whatever": 2617,
+  "When": 313,
+  "Where": 476,
+  "Where's": 1654,
+  "Which": 1271,
+  "While": 2486,
+  "Who": 506,
+  "Who's": 2026,
+  "Whoa": 2723,
+  "Why": 235,
+  "Will": 1277,
+  "With": 876,
+  "Without": 3854,
+  "Would": 1472,
+  "Wow": 1760,
+  "YOU": 2480,
+  "Yeah": 123,
+  "Yep": 3612,
+  "Yes": 178,
+  "You": 57,
+  "You'd": 3187,
+  "You'll": 1130,
+  "You're": 241,
+  "You've": 947,
+  "Your": 493,
+  "[": 177,
+  "]": 202,
+  "] ": 1070,
+  "_": 474,
+  "`": 661,
+  "``": 244,
+  "a": 492,
+  "about": 3837,
+  "ah": 3099,
+  "aha": 1426,
+  "all": 1695,
+  "alright": 2296,
+  "also": 3155,
+  "amp": 306,
+  "an": 2882,
+  "and": 172,
+  "are": 1145,
+  "as": 1624,
+  "asked": 3728,
+  "at": 2282,
+  "be": 2820,
+  "because": 1837,
+  "born": 625,
+  "br": 1261,
+  "but": 595,
+  "by": 2967,
+  "can": 971,
+  "come": 1471,
+  "d": 3253,
+  "did": 1325,
+  "do": 762,
+  "does": 3281,
+  "don't": 1843,
+  "e": 3101,
+  "for": 1218,
+  "from": 2231,
+  "get": 3533,
+  "go": 2818,
+  "good": 1946,
+  "gt": 481,
+  "had": 2438,
+  "have": 2247,
+  "he": 789,
+  "he's": 2069,
+  "hello": 3981,
+  "her": 2493,
+  "here": 1199,
+  "here's": 2999,
+  "hey": 2825,
+  "hi": 3617,
+  "him": 3764,
+  "his": 1785,
+  "hm": 1701,
+  "how": 1123,
+  "huh": 2305,
+  "i": 326,
+  "i'll": 2861,
+  "i'm": 1714,
+  "if": 1727,
+  "in": 738,
+  "is": 642,
+  "it": 730,
+  "it's": 706,
+  "just": 2146,
+  "km": 2709,
+  "let": 2150,
+  "let's": 1213,
+  "like": 1616,
+  "little": 3562,
+  "ll": 2676,
+  "look": 1093,
+  "lt": 1121,
+  "m": 1446,
+  "man": 4000,
+  "maybe": 3527,
+  "mhm": 1637,
+  "mm": 1551,
+  "more": 3550,
+  "my": 2362,
+  "nd": 2784,
+  "no": 477,
+  "not": 1504,
+  "now": 1205,
+  "of": 702,
+  "oh": 295,
+  "okay": 411,
+  "on": 1779,
+  "one": 1716,
+  "or": 1059,
+  "put": 2333,
+  "rd": 3171,
+  "re": 3243,
+  "right": 1434,
+  "s": 219,
+  "said": 2235,
+  "say": 3885,
+  "see": 1414,
+  "she": 1220,
+  "she's": 3627,
+  "should": 3597,
+  "so": 926,
+  "some": 3778,
+  "st": 2256,
+  "t": 831,
+  "th": 365,
+  "thank": 2623,
+  "that": 592,
+  "that's": 486,
+  "the": 221,
+  "their": 3795,
+  "then": 2474,
+  "there": 972,
+  "there's": 1766,
+  "they": 1187,
+  "they're": 3236,
+  "this": 1006,
+  "those": 3941,
+  "to": 705,
+  "two": 3147,
+  "uh": 2464,
+  "up": 3996,
+  "ve": 3833,
+  "very": 2938,
+  "want": 3447,
+  "was": 1369,
+  "we": 978,
+  "we'll": 4026,
+  "we're": 3905,
+  "well": 837,
+  "were": 3037,
+  "what": 280,
+  "what's": 746,
+  "when": 2015,
+  "where": 1118,
+  "where's": 1708,
+  "which": 1324,
+  "who": 1385,
+  "who's": 2691,
+  "why": 1587,
+  "will": 3430,
+  "with": 1212,
+  "would": 1910,
+  "wow": 3816,
+  "x": 3002,
+  "yeah": 429,
+  "yes": 1016,
+  "you": 232,
+  "you're": 1171,
+  "your": 2787,
+  "’": 3567,
+  " ": 2647,
+  "´": 1304,
+  "¶": 3573,
+  "–": 1455,
+  "—": 979,
+  "’": 239,
+  "“": 327,
+  "”": 2161,
+  "♪": 413,
+  "♫": 3413
+}