T5Tokenizer

backend/services/QuestionGenerator.py

@@ -1,4 +1,4 @@
-from transformers import pipeline, AutoTokenizer, AutoModelForSeq2SeqLM
+from transformers import pipeline, T5Tokenizer, AutoModelForSeq2SeqLM
 from .IQuestionGenerator import IQuestionGenerator
 from backend.services.SentenceCheck import SentenceCheck
 from backend.models.AIParamModel import AIParam
@@ -8,7 +8,7 @@ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 print(f"[QuestionGenerator] Using device: {device}")
 
 # valhalla model with slow tokenizer
-tokenizer_qg_simple = AutoTokenizer.from_pretrained("valhalla/t5-small-qg-hl", use_fast=False)
+tokenizer_qg_simple = T5Tokenizer.from_pretrained("valhalla/t5-small-qg-hl")
 model_qg_simple = AutoModelForSeq2SeqLM.from_pretrained("valhalla/t5-small-qg-hl")
 
 qg_simple = pipeline(
@@ -19,7 +19,7 @@ qg_simple = pipeline(
 )
 
 # iarfmoose model with slow tokenizer
-tokenizer_qg_advanced = AutoTokenizer.from_pretrained("iarfmoose/t5-base-question-generator", use_fast=False)
+tokenizer_qg_advanced = T5Tokenizer.from_pretrained("iarfmoose/t5-base-question-generator")
 model_qg_advanced = AutoModelForSeq2SeqLM.from_pretrained("iarfmoose/t5-base-question-generator")
 
 qg_advanced = pipeline(