SecureLLMSys commited on
Commit
c9b5c5a
·
1 Parent(s): 6cf8d11
Files changed (1) hide show
  1. app.py +3 -2
app.py CHANGED
@@ -23,6 +23,7 @@ os.makedirs("/home/user/nltk_data", exist_ok=True)
23
  nltk.download("punkt", download_dir="/home/user/nltk_data")
24
  # Tell nltk where to find it
25
  nltk.data.path.append("/home/user/nltk_data")
 
26
 
27
  # Load original app constants
28
  APP_TITLE = '<div class="app-title"><span class="brand">AttnTrace: </span><span class="subtitle">Attention-based Context Traceback for Long-Context LLMs</span></div>'
@@ -234,8 +235,8 @@ def split_into_sentences(text: str):
234
  lines = text.splitlines()
235
  sentences = []
236
  for line in lines:
237
- #sentences.extend(nltk.sent_tokenize(line))
238
- sentences.extend(line.split("."))
239
  separators = []
240
  cur_start = 0
241
  for sentence in sentences:
 
23
  nltk.download("punkt", download_dir="/home/user/nltk_data")
24
  # Tell nltk where to find it
25
  nltk.data.path.append("/home/user/nltk_data")
26
+ from nltk.tokenize import sent_tokenize
27
 
28
  # Load original app constants
29
  APP_TITLE = '<div class="app-title"><span class="brand">AttnTrace: </span><span class="subtitle">Attention-based Context Traceback for Long-Context LLMs</span></div>'
 
235
  lines = text.splitlines()
236
  sentences = []
237
  for line in lines:
238
+ sentences.extend(sent_tokenize(line))
239
+ #sentences.extend(line.split("."))
240
  separators = []
241
  cur_start = 0
242
  for sentence in sentences: