Update app.py
Browse files
app.py
CHANGED
|
@@ -708,7 +708,6 @@ def sentence_splitter_instantiation(
|
|
| 708 |
):
|
| 709 |
### Chunker/Sentence Splitter
|
| 710 |
def simple_whitespace_tokenizer(text):
|
| 711 |
-
"""Tokenizer that considers each word as one token"""
|
| 712 |
return text.split()
|
| 713 |
|
| 714 |
if sentence_splitter_config.value is not None:
|
|
|
|
| 708 |
):
|
| 709 |
### Chunker/Sentence Splitter
|
| 710 |
def simple_whitespace_tokenizer(text):
|
|
|
|
| 711 |
return text.split()
|
| 712 |
|
| 713 |
if sentence_splitter_config.value is not None:
|