decoding tokens

app.py

@@ -6,11 +6,11 @@ tokenizer = BPETokenizer()
 tokenizer.load('models/english_5000.model')
 
 def inference(input_text):
-    tokens = tokenizer.encode_ordinary(input_text)
-    
-    return len(tokens), tokens
+    encoding = tokenizer.encode_ordinary(input_text)
+    sentence = tokenizer.decode(encoding)
+    return len(encoding), sentence, encoding
 
-title = "A bilingual tokenizer build using opus and wikipedia data"
+title = "Bilingual tokenizer"
 description = "A simple Gradio interface to see tokenization of Hindi and English(Hinglish) text"
 examples = [["He walked into the basement with the horror movie from the night before playing in his head."], 
            ["Henry couldn't decide if he was an auto mechanic or a priest."], 
@@ -23,7 +23,8 @@ demo = gr.Interface(
         ], 
     outputs = [
         gr.Label(label="Token count"),
-        gr.Textbox(label="Output", type="text")
+        gr.Textbox(label="Sentence after tokenization", type="text"),
+        gr.Textbox(label="Encoding", type="text")
         ],
     title = title,
     description = description,