Update app.py

app.py

@@ -1,22 +1,23 @@
 import json
-from transformers import pipeline
+from transformers import pipeline, AutoModelForSeq2SeqLM, T5Tokenizer, AutoTokenizer
 import gradio as gr
 
 # Load question-generation and question-answering pipelines
-# Use 'text2text-generation' for QG since 'e2e-qg' is not a recognized task
+# Use T5Tokenizer with use_fast=False to avoid tiktoken dependency
+qg_model = AutoModelForSeq2SeqLM.from_pretrained("valhalla/t5-small-qa-qg-hl")
+qg_tokenizer = T5Tokenizer.from_pretrained("valhalla/t5-small-qa-qg-hl", use_fast=False)
 qg_pipeline = pipeline(
-    "text2text-generation", 
-    model="valhalla/t5-small-qa-qg-hl",
-    tokenizer="valhalla/t5-small-qa-qg-hl"
+    "text2text-generation",
+    model=qg_model,
+    tokenizer=qg_tokenizer
 )
-qa_pipeline = pipeline("question-answering", model="distilbert-base-cased-distilled-squad")
+qa_pipeline = pipeline("question-answering", model="distilbert-base-cased-distilled-squad", tokenizer="distilbert-base-cased-distilled-squad")
 
 # Simple chunking: split on paragraphs (for demo)
 def split_chunks(text, max_len=200):
     paragraphs = [p.strip() for p in text.split("\n") if p.strip()]
     chunks = []
     for p in paragraphs:
-        # further split long paragraphs
         words = p.split()
         if len(words) <= max_len:
             chunks.append(p)
@@ -31,21 +32,18 @@ def convert_text(raw_text):
     chunks = split_chunks(raw_text)
     qna_list = []
     for chunk in chunks:
-        # Generate raw Q&A pairs
         try:
-            # The model expects a prompt prefix for QG
             prompt = f"generate question: {chunk}"
             outputs = qg_pipeline(prompt, max_length=64, clean_up_tokenization_spaces=True)
         except Exception:
             continue
         for out in outputs:
-            question = out["generated_text"].strip()
+            question = out.get("generated_text", out.get("text", "")).strip()
             if not question.endswith("?"):
                 question += "?"
             # Refine answer using QA pipeline
             ans = qa_pipeline({"question": question, "context": chunk})
             answer = ans.get("answer", "").strip()
-            # Append result
             qna_list.append({"question": question, "answer": answer})
     # Deduplicate
     unique = []