Update tasks/text.py

tasks/text.py

@@ -7,10 +7,12 @@ import random
 from .utils.evaluation import TextEvaluationRequest
 from .utils.emissions import tracker, clean_emissions_data, get_space_info
 
-from transformers import AutoTokenizer, AutoModelForSequenceClassification
+from transformers import AutoTokenizer, AutoModelForSequenceClassification, MobileBertTokenizerFast
 
 import numpy as np
 import torch
+from transformers import MobileBertTokenizerFast
+
 
 
 router = APIRouter()
@@ -63,7 +65,6 @@ async def evaluate_text(request: TextEvaluationRequest):
 
     # Model and Tokenizer
     model_name = "Zen0/FrugalDisinfoHunter" 
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
     model = AutoModelForSequenceClassification.from_pretrained("Zen0/FrugalDisinfoHunter", num_labels=8)
     
     # Tokenize the test data