Update app.py

app.py

@@ -90,16 +90,16 @@ def load_json_data(json_path):
         return ""
 
 # ---------------- Chunking ----------------
-def chunk_text(text, tokenizer, chunk_size=512, chunk_overlap=50):
-    input_ids = tokenizer.encode(text, add_special_tokens=False)
+def chunk_text(text, tokenizer, chunk_size=128, chunk_overlap=32, max_tokens=512):
+    tokens = tokenizer.tokenize(text)
     chunks = []
     start = 0
-    while start < len(input_ids):
-        end = min(start + chunk_size, len(input_ids))
-        chunk_ids = input_ids[start:end]
-        chunk_text = tokenizer.decode(chunk_ids)
+    while start < len(tokens):
+        end = min(start + chunk_size, len(tokens))
+        chunk_tokens = tokens[start:end]
+        chunk_text = tokenizer.convert_tokens_to_string(chunk_tokens)
         chunks.append(chunk_text)
-        if end == len(input_ids):
+        if end == len(tokens):
             break
         start += chunk_size - chunk_overlap
     return chunks
@@ -201,7 +201,7 @@ def setup_knowledge_base():
             except Exception as e:
                 print(f"CSV read error: {e}")
 
-    tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
+    tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased', model_max_length=512)
     chunks = chunk_text(all_text, tokenizer)
     model = SentenceTransformer('all-mpnet-base-v2')
     embeddings = model.encode(chunks, show_progress_bar=False)