Synced repo using 'sync_with_huggingface' Github Action
Browse files
src/data_utils/text_processor.py
CHANGED
|
@@ -28,6 +28,7 @@ class TextProcessor:
|
|
| 28 |
word_tokenize("test")
|
| 29 |
except LookupError:
|
| 30 |
nltk.download("punkt")
|
|
|
|
| 31 |
|
| 32 |
|
| 33 |
def preprocess_text(self, text: str) -> List[str]:
|
|
|
|
| 28 |
word_tokenize("test")
|
| 29 |
except LookupError:
|
| 30 |
nltk.download("punkt")
|
| 31 |
+
nltk.download('punkt_tab')
|
| 32 |
|
| 33 |
|
| 34 |
def preprocess_text(self, text: str) -> List[str]:
|