Update app.py

app.py

@@ -90,18 +90,21 @@ def load_json_data(json_path):
         return ""
 
 # ---------------- Chunking ----------------
-def chunk_text(text, tokenizer, chunk_size=128, chunk_overlap=32):
-    tokens = tokenizer.tokenize(text)
+def chunk_text(text, tokenizer, chunk_size=512, chunk_overlap=50):
+    input_ids = tokenizer.encode(text, add_special_tokens=False)
     chunks = []
     start = 0
-    while start < len(tokens):
-        end = min(start + chunk_size, len(tokens))
-        chunk = tokens[start:end]
-        chunks.append(tokenizer.convert_tokens_to_string(chunk))
-        if end == len(tokens): break
+    while start < len(input_ids):
+        end = min(start + chunk_size, len(input_ids))
+        chunk_ids = input_ids[start:end]
+        chunk_text = tokenizer.decode(chunk_ids)
+        chunks.append(chunk_text)
+        if end == len(input_ids):
+            break
         start += chunk_size - chunk_overlap
     return chunks
 
+
 def retrieve_chunks(question, index, embed_model, text_chunks, k=3):
     q_embedding = embed_model.encode(question)
     D, I = index.search(np.array([q_embedding]), k)