Spaces:
Running
on
Zero
Running
on
Zero
Commit
·
c9b5c5a
1
Parent(s):
6cf8d11
update
Browse files
app.py
CHANGED
@@ -23,6 +23,7 @@ os.makedirs("/home/user/nltk_data", exist_ok=True)
|
|
23 |
nltk.download("punkt", download_dir="/home/user/nltk_data")
|
24 |
# Tell nltk where to find it
|
25 |
nltk.data.path.append("/home/user/nltk_data")
|
|
|
26 |
|
27 |
# Load original app constants
|
28 |
APP_TITLE = '<div class="app-title"><span class="brand">AttnTrace: </span><span class="subtitle">Attention-based Context Traceback for Long-Context LLMs</span></div>'
|
@@ -234,8 +235,8 @@ def split_into_sentences(text: str):
|
|
234 |
lines = text.splitlines()
|
235 |
sentences = []
|
236 |
for line in lines:
|
237 |
-
|
238 |
-
sentences.extend(line.split("."))
|
239 |
separators = []
|
240 |
cur_start = 0
|
241 |
for sentence in sentences:
|
|
|
23 |
nltk.download("punkt", download_dir="/home/user/nltk_data")
|
24 |
# Tell nltk where to find it
|
25 |
nltk.data.path.append("/home/user/nltk_data")
|
26 |
+
from nltk.tokenize import sent_tokenize
|
27 |
|
28 |
# Load original app constants
|
29 |
APP_TITLE = '<div class="app-title"><span class="brand">AttnTrace: </span><span class="subtitle">Attention-based Context Traceback for Long-Context LLMs</span></div>'
|
|
|
235 |
lines = text.splitlines()
|
236 |
sentences = []
|
237 |
for line in lines:
|
238 |
+
sentences.extend(sent_tokenize(line))
|
239 |
+
#sentences.extend(line.split("."))
|
240 |
separators = []
|
241 |
cur_start = 0
|
242 |
for sentence in sentences:
|