Update app.py

app.py

@@ -1,7 +1,6 @@
-from PyPDF2 import PdfFileReader
-from tiktoken import Tokenizer
-from tiktoken.models import GPT2
 import gradio as gr
+from PyPDF4 import PdfFileReader
+import tiktoken
 
 def extract_text_from_pdf(file_path):
     with open(file_path, "rb") as file:
@@ -12,8 +11,11 @@ def extract_text_from_pdf(file_path):
     return text
 
 def count_tokens(text):
-    tokenizer = Tokenizer(GPT2())
-    tokens = tokenizer.tokenize(text)
+    tokenizer = tiktoken.encoding_for_model("gpt-3.5-turbo")
+    tokens = tokenizer.encode(
+        text,
+        disallowed_special=()
+    )
     return len(tokens)
 
 def count_tokens_in_file(file):