move preprocessor

tasks/text.py

@@ -19,37 +19,7 @@ models_description = {
     "tfidf_xgb": "TF-IDF vectorizer and XGBoost classifier",
 }
 
-# Textpreprocessor 
-import nltk
-from nltk.stem import WordNetLemmatizer
-from sklearn.base import BaseEstimator, TransformerMixin
-import nltk
-import contractions
-
-# Download required NLTK resources
-nltk.download('punkt_tab')
-nltk.download('wordnet')
-
-# Custom sklearn transformer for preprocessing text
-class TextPreprocessor(BaseEstimator, TransformerMixin):
-    def __init__(self):
-        self.lemmatizer = WordNetLemmatizer()
-
-    def fit(self, X, y=None):
-        return self
-
-    def transform(self, X):
-        preprocessed_texts = []
-        for doc in X:
-            # Expand contractions
-            expanded = contractions.fix(doc)
-            # Lowercase
-            lowered = expanded.lower()
-
-            # Tokenize and lemmatize
-            lemmatized = " ".join([self.lemmatizer.lemmatize(word) for word in nltk.word_tokenize(lowered)])
-            preprocessed_texts.append(lemmatized)
-        return preprocessed_texts
+
 
 
 # Some code borrowed from Nonnormalizable
@@ -61,6 +31,37 @@ def baseline_model(dataset_length: int):
     return predictions
 
 def tree_classifier(test_dataset: dict, model: str):
+    # Textpreprocessor defined in this scope
+    import nltk
+    from nltk.stem import WordNetLemmatizer
+    from sklearn.base import BaseEstimator, TransformerMixin
+    import nltk
+    import contractions
+
+    # Download required NLTK resources
+    nltk.download('punkt_tab')
+    nltk.download('wordnet')
+
+    # Custom sklearn transformer for preprocessing text
+    class TextPreprocessor(BaseEstimator, TransformerMixin):
+        def __init__(self):
+            self.lemmatizer = WordNetLemmatizer()
+
+        def fit(self, X, y=None):
+            return self
+
+        def transform(self, X):
+            preprocessed_texts = []
+            for doc in X:
+                # Expand contractions
+                expanded = contractions.fix(doc)
+                # Lowercase
+                lowered = expanded.lower()
+
+                # Tokenize and lemmatize
+                lemmatized = " ".join([self.lemmatizer.lemmatize(word) for word in nltk.word_tokenize(lowered)])
+                preprocessed_texts.append(lemmatized)
+            return preprocessed_texts
 
     texts = test_dataset["quote"]