more chunking updates

app.py

@@ -24,13 +24,20 @@ class SummarizationResponseItem(BaseModel):
 class BatchSummarizationResponse(BaseModel):
     summaries: List[SummarizationResponseItem]
 
-def chunk_text(text, max_tokens=700):
+# Ensure no chunk ever exceeds model token limit
+MAX_MODEL_TOKENS = 1024
+SAFE_CHUNK_SIZE = 700
+
+def chunk_text(text: str, max_tokens: int = SAFE_CHUNK_SIZE) -> List[str]:
     tokens = tokenizer.encode(text, truncation=False)
     chunks = []
 
     for i in range(0, len(tokens), max_tokens):
-        chunk = tokens[i:i + max_tokens]
-        chunks.append(tokenizer.decode(chunk, skip_special_tokens=True))
+        chunk_tokens = tokens[i:i + max_tokens]
+        if len(chunk_tokens) > MAX_MODEL_TOKENS:
+            chunk_tokens = chunk_tokens[:MAX_MODEL_TOKENS]
+        chunk = tokenizer.decode(chunk_tokens, skip_special_tokens=True)
+        chunks.append(chunk)
 
     return chunks
 
@@ -44,8 +51,14 @@ async def summarize_batch(request: BatchSummarizationRequest):
         all_chunks.extend(chunks)
         chunk_map.extend([item.content_id] * len(chunks))
 
+    # Final safety pass to enforce 1024 token limit
+    safe_chunks = [
+        tokenizer.decode(tokenizer.encode(chunk, truncation=False)[:MAX_MODEL_TOKENS], skip_special_tokens=True)
+        for chunk in all_chunks
+    ]
+
     summaries = summarizer(
-        all_chunks,
+        safe_chunks,
         max_length=150,
         min_length=30,
         truncation=True,