why

app.py

@@ -39,8 +39,6 @@ def chunk_text(text: str, max_tokens: int = SAFE_CHUNK_SIZE) -> List[str]:
 
     for i in range(0, len(tokens), max_tokens):
         chunk_tokens = tokens[i:i + max_tokens]
-        if len(chunk_tokens) > MAX_MODEL_TOKENS:
-            chunk_tokens = chunk_tokens[:MAX_MODEL_TOKENS]
         chunk = tokenizer.decode(chunk_tokens, skip_special_tokens=True)
         chunks.append(chunk)
 
@@ -57,11 +55,14 @@ async def summarize_batch(request: BatchSummarizationRequest):
         all_chunks.extend(chunks)
         chunk_map.extend([item.content_id] * len(chunks))
 
-    # Final safety pass to enforce 1024 token limit
-    safe_chunks = [
-        tokenizer.decode(tokenizer.encode(chunk, truncation=False)[:MAX_MODEL_TOKENS], skip_special_tokens=True)
-        for chunk in all_chunks
-    ]
+    # Final safety pass to enforce 1024 token limit after decoding
+    safe_chunks = []
+    for chunk in all_chunks:
+        encoded = tokenizer.encode(chunk, truncation=False)
+        if len(encoded) > MAX_MODEL_TOKENS:
+            logger.warning(f"[TRUNCATING] Chunk exceeded max tokens ({len(encoded)}), trimming to {MAX_MODEL_TOKENS} tokens")
+            encoded = encoded[:MAX_MODEL_TOKENS]
+        safe_chunks.append(tokenizer.decode(encoded, skip_special_tokens=True))
 
     summaries = summarizer(
         safe_chunks,