Spaces:
Sleeping
Sleeping
Commit
·
b48fa68
1
Parent(s):
d872090
Update app.py
Browse files
app.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
import transformers
|
| 2 |
import streamlit as st
|
| 3 |
-
|
| 4 |
from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
|
| 5 |
import json
|
| 6 |
|
|
@@ -33,6 +33,7 @@ def infer(input_ids, max_length, temperature, top_k, top_p):
|
|
| 33 |
|
| 34 |
|
| 35 |
def chunking(book_text):
|
|
|
|
| 36 |
segments = []
|
| 37 |
#sentences, token_lens
|
| 38 |
current_segment = ""
|
|
|
|
| 1 |
import transformers
|
| 2 |
import streamlit as st
|
| 3 |
+
from nltk import sent_tokenize
|
| 4 |
from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
|
| 5 |
import json
|
| 6 |
|
|
|
|
| 33 |
|
| 34 |
|
| 35 |
def chunking(book_text):
|
| 36 |
+
sentences = sent_tokenize(book_text)
|
| 37 |
segments = []
|
| 38 |
#sentences, token_lens
|
| 39 |
current_segment = ""
|