what

app.py

@@ -39,6 +39,8 @@ def chunk_text(text: str, max_tokens: int = SAFE_CHUNK_SIZE) -> List[str]:
 
     for i in range(0, len(tokens), max_tokens):
         chunk_tokens = tokens[i:i + max_tokens]
+        if len(chunk_tokens) > MAX_MODEL_TOKENS:
+            chunk_tokens = chunk_tokens[:MAX_MODEL_TOKENS]
         chunk = tokenizer.decode(chunk_tokens, skip_special_tokens=True)
         chunks.append(chunk)
 
@@ -56,25 +58,12 @@ async def summarize_batch(request: BatchSummarizationRequest):
         all_chunks.extend(chunks)
         chunk_map.extend([item.content_id] * len(chunks))
 
-    # Retokenize and only allow chunks that are safely below the max token limit
-    safe_chunks = []
-    safe_chunk_map = []
-    for content_id, chunk in zip(chunk_map, all_chunks):
-        encoded = tokenizer(chunk, return_tensors="pt", truncation=True, max_length=MAX_MODEL_TOKENS)
-        token_count = encoded["input_ids"].shape[1]
-        if token_count > MAX_MODEL_TOKENS:
-            logger.warning(f"[SKIP] content_id={content_id} Chunk too long after truncation: {token_count} tokens")
-            continue
-        decoded = tokenizer.decode(encoded["input_ids"][0], skip_special_tokens=True)
-        safe_chunks.append(decoded)
-        safe_chunk_map.append(content_id)
-
-    if not safe_chunks:
-        logger.error("No valid chunks after token filtering. Returning empty response.")
+    if not all_chunks:
+        logger.error("No valid chunks after chunking. Returning empty response.")
         return {"summaries": []}
 
     summaries = summarizer(
-        safe_chunks,
+        all_chunks,
         max_length=150,
         min_length=30,
         truncation=True,
@@ -84,7 +73,7 @@ async def summarize_batch(request: BatchSummarizationRequest):
 
     # Aggregate summaries back per content_id
     summary_map = {}
-    for content_id, result in zip(safe_chunk_map, summaries):
+    for content_id, result in zip(chunk_map, summaries):
         summary_map.setdefault(content_id, []).append(result["summary_text"])
 
     response_items = [