Hmm

app.py

@@ -42,6 +42,7 @@ def predict(code):
     """
 
     print(f"Prompt: {prompt}")
+    print(f"Tokenized: {tokenizer.tokenize(prompt)}")
     return pipe(prompt)
 
 demo = gr.Interface(fn=predict, inputs="text", outputs="text",