Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
@@ -32,24 +32,8 @@ length_function_option = st.selectbox(
|
|
32 |
["Characters", "Tokens"]
|
33 |
)
|
34 |
|
35 |
-
splitter_choice = st.selectbox(
|
36 |
-
"Select a Text Splitter",
|
37 |
-
["RecursiveCharacter", "Character"]
|
38 |
-
)
|
39 |
-
# Auswählen der passenden length_function
|
40 |
-
if length_function_option == "Characters":
|
41 |
-
# Messen in Zeichen
|
42 |
-
length_function = len
|
43 |
-
length_function_str = CHARACTER_LENGTH
|
44 |
-
elif length_function_option == "Tokens":
|
45 |
-
# Messen in Tokens mithilfe tiktoken
|
46 |
-
enc = tiktoken.get_encoding("cl100k_base")
|
47 |
-
def length_function(text: str) -> int:
|
48 |
-
return len(enc.encode(text))
|
49 |
-
length_function_str = TOKEN_LENGTH
|
50 |
-
else:
|
51 |
-
raise ValueError("Ungültige Option für length_function.")
|
52 |
|
|
|
53 |
|
54 |
# Text-Eingabe
|
55 |
doc = st.text_area("Füge hier deinen Text ein:")
|
|
|
32 |
["Characters", "Tokens"]
|
33 |
)
|
34 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
35 |
|
36 |
+
length_function_option = Tokens
|
37 |
|
38 |
# Text-Eingabe
|
39 |
doc = st.text_area("Füge hier deinen Text ein:")
|