move text preprocessor into app

tasks/text.py

@@ -4,7 +4,6 @@ from datasets import load_dataset
 from sklearn.metrics import accuracy_score
 import random
 from skops.io import load
-from .utils.text_preprocessor import TextPreprocessor
 
 
 from .utils.evaluation import TextEvaluationRequest
@@ -20,6 +19,39 @@ models_description = {
     "tfidf_xgb": "TF-IDF vectorizer and XGBoost classifier",
 }
 
+# Textpreprocessor 
+import nltk
+from nltk.stem import WordNetLemmatizer
+from sklearn.base import BaseEstimator, TransformerMixin
+import nltk
+import contractions
+
+# Download required NLTK resources
+nltk.download('punkt_tab')
+nltk.download('wordnet')
+
+# Custom sklearn transformer for preprocessing text
+class TextPreprocessor(BaseEstimator, TransformerMixin):
+    def __init__(self):
+        self.lemmatizer = WordNetLemmatizer()
+
+    def fit(self, X, y=None):
+        return self
+
+    def transform(self, X):
+        preprocessed_texts = []
+        for doc in X:
+            # Expand contractions
+            expanded = contractions.fix(doc)
+            # Lowercase
+            lowered = expanded.lower()
+
+            # Tokenize and lemmatize
+            lemmatized = " ".join([self.lemmatizer.lemmatize(word) for word in nltk.word_tokenize(lowered)])
+            preprocessed_texts.append(lemmatized)
+        return preprocessed_texts
+
+
 # Some code borrowed from Nonnormalizable
 
 def baseline_model(dataset_length: int):
@@ -36,7 +68,7 @@ def tree_classifier(test_dataset: dict, model: str):
 
     model = load(model_path, 
                  trusted=[
-                     'utils.text_preprocessor.TextPreprocessor',
+                     '__main__.TextPreprocessor',
                      'nltk.stem.wordnet.WordNetLemmatizer',
                      'xgboost.core.Booster',
                      'xgboost.sklearn.XGBClassifier'])

tasks/utils/text_preprocessor.py

@@ -1,30 +0,0 @@
-import nltk
-from nltk.stem import WordNetLemmatizer
-from sklearn.base import BaseEstimator, TransformerMixin
-import nltk
-import contractions
-
-# Download required NLTK resources
-nltk.download('punkt_tab')
-nltk.download('wordnet')
-
-# Custom transformer for preprocessing text
-class TextPreprocessor(BaseEstimator, TransformerMixin):
-    def __init__(self):
-        self.lemmatizer = WordNetLemmatizer()
-
-    def fit(self, X, y=None):
-        return self  # Does nothing, just returns the instance
-
-    def transform(self, X):
-        preprocessed_texts = []
-        for doc in X:
-            # Expand contractions
-            expanded = contractions.fix(doc)
-            # Lowercase
-            lowered = expanded.lower()
-
-            # Tokenize and lemmatize
-            lemmatized = " ".join([self.lemmatizer.lemmatize(word) for word in nltk.word_tokenize(lowered)])
-            preprocessed_texts.append(lemmatized)
-        return preprocessed_texts