Update app.py

app.py

@@ -36,6 +36,10 @@ def infer(input_ids, max_length, temperature, top_k, top_p):
 def chunking(book_text):
     sentences = sent_tokenize(book_text)
     segments = [] 
+    token_lens = []
+
+    for sent_i_th in sentences:
+        token_lens.append(len(tokenizer.tokenize(sent_i_th)))
     #sentences, token_lens
     current_segment = ""    
     total_token_lens = 0