Update app.py

app.py

@@ -35,16 +35,22 @@ def extract_text_from_docx(docx_path):
         return ""
 
 # --- Chunking & Retrieval ---
-def chunk_text(text, tokenizer, chunk_size=150, chunk_overlap=30):
+def chunk_text(text, tokenizer, chunk_size=128, chunk_overlap=32, max_tokens=512):
     tokens = tokenizer.tokenize(text)
-    chunks, start = [], 0
+    chunks = []
+    start = 0
     while start < len(tokens):
         end = min(start + chunk_size, len(tokens))
         chunk_tokens = tokens[start:end]
-        chunks.append(tokenizer.convert_tokens_to_string(chunk_tokens))
+        # Drop chunk if it's too long after detokenization
+        chunk_text = tokenizer.convert_tokens_to_string(chunk_tokens)
+        # Double-check token count with tokenizer to be safe
+        if len(tokenizer.encode(chunk_text)) <= max_tokens:
+            chunks.append(chunk_text.strip())
         start += chunk_size - chunk_overlap
     return chunks
 
+
 def retrieve_chunks(question, index, embed_model, text_chunks, k=3):
     question_embedding = embed_model.encode(question)
     D, I = index.search(np.array([question_embedding]), k)