Sebbe33 commited on
Commit
1d8c55e
·
verified ·
1 Parent(s): 17b7b74

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +2 -2
app.py CHANGED
@@ -4,10 +4,10 @@ import tiktoken
4
 
5
 
6
  CHARACTER_LENGTH = "length_function=lambda x: len(x)"
7
- TOKEN_LENGTH = """enc = tiktoken.get_encoding("cl100k_base")
8
  length_function = lambda text: len(enc.encode(text))
9
 
10
- RECURSIVE_CHARACTER = """RecursiveCharacterTextSplitter(
11
  chunk_size={chunk_size},
12
  chunk_overlap={chunk_overlap},
13
  length_function={length_function}
 
4
 
5
 
6
  CHARACTER_LENGTH = "length_function=lambda x: len(x)"
7
+ TOKEN_LENGTH = enc = tiktoken.get_encoding("cl100k_base")
8
  length_function = lambda text: len(enc.encode(text))
9
 
10
+ RECURSIVE_CHARACTER = RecursiveCharacterTextSplitter(
11
  chunk_size={chunk_size},
12
  chunk_overlap={chunk_overlap},
13
  length_function={length_function}